diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d96c5a0..39d8dc38 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -153,7 +153,7 @@ workflows: context: [ qqq-maven-registry-credentials, build-qqq-sample-app ] filters: branches: - ignore: /dev/ + ignore: /(dev|integration.*)/ tags: ignore: /(version|snapshot)-.*/ @@ -163,7 +163,7 @@ workflows: context: [ qqq-maven-registry-credentials, build-qqq-sample-app ] filters: branches: - only: /dev/ + only: /(dev|integration.*)/ tags: only: /(version|snapshot)-.*/ - publish_asciidoc: diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index 38bad7f5..26a4848f 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -102,6 +102,16 @@ fastexcel 0.12.15 + + org.apache.poi + poi + 5.2.5 + + + org.apache.poi + poi-ooxml + 5.2.5 + com.auth0 auth0 diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java index 6cc317d5..875097fa 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java @@ -169,17 +169,24 @@ public class AsyncJobManager LOG.debug("Completed job " + uuidAndTypeStateKey.getUuid()); return (result); } - catch(Exception e) + catch(Throwable t) { asyncJobStatus.setState(AsyncJobState.ERROR); - asyncJobStatus.setCaughtException(e); + if(t instanceof Exception e) + { + asyncJobStatus.setCaughtException(e); + } + else + { + asyncJobStatus.setCaughtException(new QException("Caught throwable", t)); + } getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus); ////////////////////////////////////////////////////// // if user facing, just log an info, warn otherwise // ////////////////////////////////////////////////////// - LOG.log((e instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid())); - throw (new CompletionException(e)); + LOG.log((t instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", t, logPair("jobId", uuidAndTypeStateKey.getUuid())); + throw (new CompletionException(t)); } finally { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java index 3ec54516..c5cf95f3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java @@ -31,6 +31,7 @@ import java.io.Serializable; *******************************************************************************/ public class AsyncJobStatus implements Serializable { + private String jobName; private AsyncJobState state; private String message; private Integer current; @@ -187,4 +188,36 @@ public class AsyncJobStatus implements Serializable { this.cancelRequested = cancelRequested; } + + + + /******************************************************************************* + ** Getter for jobName + *******************************************************************************/ + public String getJobName() + { + return (this.jobName); + } + + + + /******************************************************************************* + ** Setter for jobName + *******************************************************************************/ + public void setJobName(String jobName) + { + this.jobName = jobName; + } + + + + /******************************************************************************* + ** Fluent setter for jobName + *******************************************************************************/ + public AsyncJobStatus withJobName(String jobName) + { + this.jobName = jobName; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/NonPersistedAsyncJobCallback.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/NonPersistedAsyncJobCallback.java new file mode 100644 index 00000000..2373dc79 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/NonPersistedAsyncJobCallback.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.async; + + +import java.util.UUID; + + +/******************************************************************************* + ** subclass designed to be used when we want there to be an instance (so code + ** doesn't have to all be null-tolerant), but there's no one who will ever be + ** reading the status data, so we don't need to store the object in a + ** state provider. + *******************************************************************************/ +public class NonPersistedAsyncJobCallback extends AsyncJobCallback +{ + private final AsyncJobStatus asyncJobStatus; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public NonPersistedAsyncJobCallback(UUID jobUUID, AsyncJobStatus asyncJobStatus) + { + super(jobUUID, asyncJobStatus); + this.asyncJobStatus = asyncJobStatus; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected void storeUpdatedStatus() + { + /////////////////////////////////////////////////////////////////////////////////////// + // noop - cf. base class, which writes to persistence here (our point is, we do not) // + /////////////////////////////////////////////////////////////////////////////////////// + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java index 3ab02ce4..044a24b7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java @@ -303,7 +303,7 @@ public class DMLAuditAction extends AbstractQActionFunction>> pvsData = new ArrayList<>(); - List pvsLabels = new ArrayList<>(); - List pvsNames = new ArrayList<>(); + List>> dataList = new ArrayList<>(); + List labelList = new ArrayList<>(); + List nameList = new ArrayList<>(); List missingRequiredSelections = new ArrayList<>(); for(WidgetDropdownData dropdownData : CollectionUtils.nonNullList(metaData.getDropdowns())) { - String possibleValueSourceName = dropdownData.getPossibleValueSourceName(); - QPossibleValueSource possibleValueSource = input.getInstance().getPossibleValueSource(possibleValueSourceName); - - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // this looks complicated, but is just look for a label in the dropdown data and if found use it, // - // otherwise look for label in PVS and if found use that, otherwise just use the PVS name // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName); - pvsLabels.add(pvsLabel); - pvsNames.add(possibleValueSourceName); - - SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput(); - pvsInput.setPossibleValueSourceName(possibleValueSourceName); - - if(dropdownData.getForeignKeyFieldName() != null) + if(WidgetDropdownType.DATE_PICKER.equals(dropdownData.getType())) { - //////////////////////////////////////// - // look for an id in the query params // - //////////////////////////////////////// - Integer id = null; - if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id"))) + String name = dropdownData.getName(); + nameList.add(name); + labelList.add(dropdownData.getLabel()); + dataList.add(new ArrayList<>()); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // sure that something has been selected, and if not, display a message that a selection needs made // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(dropdownData.getIsRequired()) { - id = Integer.parseInt(input.getQueryParams().get("id")); - } - if(id != null) - { - pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria( - new QFilterCriteria( - dropdownData.getForeignKeyFieldName(), - QCriteriaOperator.EQUALS, - id))); + if(!input.getQueryParams().containsKey(name) || !StringUtils.hasContent(input.getQueryParams().get(name))) + { + missingRequiredSelections.add(dropdownData.getLabel()); + } } } - - SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput); - - List> dropdownOptionList = new ArrayList<>(); - pvsData.add(dropdownOptionList); - - ////////////////////////////////////////// - // sort results, dedupe, and add to map // - ////////////////////////////////////////// - Set exists = new HashSet<>(); - output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel())); - for(QPossibleValue possibleValue : output.getResults()) + else { - dropdownOptionList.add(MapBuilder.of( - "id", String.valueOf(possibleValue.getId()), - "label", possibleValue.getLabel() - )); - } - - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - // because we know the dropdowns and what the field names will be when something is selected, we can make // - // sure that something has been selected, and if not, display a message that a selection needs made // - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(dropdownData.getIsRequired()) - { - if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName))) + String possibleValueSourceName = dropdownData.getPossibleValueSourceName(); + if(possibleValueSourceName != null) { - missingRequiredSelections.add(pvsLabel); + QPossibleValueSource possibleValueSource = input.getInstance().getPossibleValueSource(possibleValueSourceName); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // this looks complicated, but is just look for a label in the dropdown data and if found use it, // + // otherwise look for label in PVS and if found use that, otherwise just use the PVS name // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName); + labelList.add(pvsLabel); + nameList.add(possibleValueSourceName); + + SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput(); + pvsInput.setPossibleValueSourceName(possibleValueSourceName); + + if(dropdownData.getForeignKeyFieldName() != null) + { + //////////////////////////////////////// + // look for an id in the query params // + //////////////////////////////////////// + Integer id = null; + if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id"))) + { + id = Integer.parseInt(input.getQueryParams().get("id")); + } + if(id != null) + { + pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria( + new QFilterCriteria( + dropdownData.getForeignKeyFieldName(), + QCriteriaOperator.EQUALS, + id))); + } + } + + SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput); + + List> dropdownOptionList = new ArrayList<>(); + dataList.add(dropdownOptionList); + + ////////////////////////////////////////// + // sort results, dedupe, and add to map // + ////////////////////////////////////////// + Set exists = new HashSet<>(); + output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel())); + for(QPossibleValue possibleValue : output.getResults()) + { + dropdownOptionList.add(MapBuilder.of( + "id", String.valueOf(possibleValue.getId()), + "label", possibleValue.getLabel() + )); + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // because we know the dropdowns and what the field names will be when something is selected, we can make // + // sure that something has been selected, and if not, display a message that a selection needs made // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(dropdownData.getIsRequired()) + { + if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName))) + { + missingRequiredSelections.add(pvsLabel); + } + } } } } - widgetData.setDropdownNameList(pvsNames); - widgetData.setDropdownLabelList(pvsLabels); - widgetData.setDropdownDataList(pvsData); + widgetData.setDropdownNameList(nameList); + widgetData.setDropdownLabelList(labelList); + widgetData.setDropdownDataList(dataList); //////////////////////////////////////////////////////////////////////////////// // if there are any missing required dropdowns, build up a message to display // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QStorageInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QStorageInterface.java new file mode 100644 index 00000000..1181494b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QStorageInterface.java @@ -0,0 +1,49 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.interfaces; + + +import java.io.InputStream; +import java.io.OutputStream; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; + + +/******************************************************************************* + ** Interface for actions that a backend can perform, based on streaming data + ** into the backend's storage. + *******************************************************************************/ +public interface QStorageInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + OutputStream createOutputStream(StorageInput storageInput) throws QException; + + + /******************************************************************************* + ** + *******************************************************************************/ + InputStream getInputStream(StorageInput storageInput) throws QException; + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java index 977104c0..eb2f24d9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java @@ -26,8 +26,15 @@ import java.io.Serializable; import java.util.Collections; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -65,4 +72,46 @@ public class QProcessCallbackFactory }; } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessCallback forRecordEntity(QRecordEntity entity) + { + return forRecord(entity.toQRecord()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessCallback forRecord(QRecord record) + { + String primaryKeyField = "id"; + if(StringUtils.hasContent(record.getTableName())) + { + primaryKeyField = QContext.getQInstance().getTable(record.getTableName()).getPrimaryKeyField(); + } + + Serializable primaryKeyValue = record.getValue(primaryKeyField); + if(primaryKeyValue == null) + { + throw (new QRuntimeException("Record did not have value in its primary key field [" + primaryKeyField + "]")); + } + + return (forPrimaryKey(primaryKeyField, primaryKeyValue)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessCallback forPrimaryKey(String fieldName, Serializable value) + { + return (forFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value)))); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java index 49ecd772..fac33e13 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -65,12 +66,12 @@ public class CsvExportStreamer implements ExportStreamerInterface ** *******************************************************************************/ @Override - public void start(ExportInput exportInput, List fields, String label) throws QReportingException + public void start(ExportInput exportInput, List fields, String label, QReportView view) throws QReportingException { this.exportInput = exportInput; this.fields = fields; table = exportInput.getTable(); - outputStream = this.exportInput.getReportOutputStream(); + outputStream = this.exportInput.getReportDestination().getReportOutputStream(); writeTitleAndHeader(); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java index 8360cdf7..e14de682 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java @@ -52,6 +52,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; @@ -138,7 +139,7 @@ public class ExportAction /////////////////////////////////////////////////////////////////////////////////////////////////////////// // check if this report format has a max-rows limit -- if so, do a count to verify we're under the limit // /////////////////////////////////////////////////////////////////////////////////////////////////////////// - ReportFormat reportFormat = exportInput.getReportFormat(); + ReportFormat reportFormat = exportInput.getReportDestination().getReportFormat(); verifyCountUnderMax(exportInput, backendModule, reportFormat); preExecuteRan = true; @@ -226,10 +227,19 @@ public class ExportAction //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // set up a report streamer, which will read rows from the pipe, and write formatted report rows to the output stream // //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - ReportFormat reportFormat = exportInput.getReportFormat(); + ReportFormat reportFormat = exportInput.getReportDestination().getReportFormat(); ExportStreamerInterface reportStreamer = reportFormat.newReportStreamer(); List fields = getFields(exportInput); - reportStreamer.start(exportInput, fields, "Sheet 1"); + + ////////////////////////////////////////////////////////// + // it seems we can pass a view with just a name in here // + ////////////////////////////////////////////////////////// + List views = new ArrayList<>(); + views.add(new QReportView() + .withName("export")); + + reportStreamer.preRun(exportInput.getReportDestination(), views); + reportStreamer.start(exportInput, fields, "Sheet 1", views.get(0)); ////////////////////////////////////////// // run the query action as an async job // @@ -318,7 +328,7 @@ public class ExportAction try { - exportInput.getReportOutputStream().close(); + exportInput.getReportDestination().getReportOutputStream().close(); } catch(Exception e) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStreamerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStreamerInterface.java index 473b3b34..4e2f7e02 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStreamerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStreamerInterface.java @@ -26,8 +26,10 @@ import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; /******************************************************************************* @@ -35,20 +37,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; *******************************************************************************/ public interface ExportStreamerInterface { - /******************************************************************************* - ** Called once, before any rows are available. Meant to write a header, for example. - *******************************************************************************/ - void start(ExportInput exportInput, List fields, String label) throws QReportingException; /******************************************************************************* - ** Called as records flow into the pipe. - ******************************************************************************/ - void addRecords(List recordList) throws QReportingException; - - /******************************************************************************* - ** Called once, after all rows are available. Meant to write a footer, or close resources, for example. + ** Called once, before any sheets are actually being produced. *******************************************************************************/ - void finish() throws QReportingException; + default void preRun(ReportDestination reportDestination, List views) throws QReportingException + { + // noop in base class + } /******************************************************************************* ** @@ -58,6 +54,20 @@ public interface ExportStreamerInterface // noop in base class } + /******************************************************************************* + ** Called once per sheet, before any rows are available. Meant to write a + ** header, for example. + ** + ** If multiple sheets are being created, there is no separate end-sheet call. + ** Rather, a new one will just get started... + *******************************************************************************/ + void start(ExportInput exportInput, List fields, String label, QReportView view) throws QReportingException; + + /******************************************************************************* + ** Called as records flow into the pipe. + ******************************************************************************/ + void addRecords(List recordList) throws QReportingException; + /******************************************************************************* ** *******************************************************************************/ @@ -65,4 +75,11 @@ public interface ExportStreamerInterface { addRecords(List.of(record)); } + + /******************************************************************************* + ** Called after all sheets are complete. Meant to do a final write, or close + ** resources, for example. + *******************************************************************************/ + void finish() throws QReportingException; + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java index 4f1fc32b..cb828123 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java @@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.actions.reporting; import java.io.Serializable; import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -32,18 +34,23 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction; import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer; import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QFormulaException; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -51,6 +58,9 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -68,12 +78,17 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.aggregates.AggregatesInterface; import com.kingsrook.qqq.backend.core.utils.aggregates.BigDecimalAggregates; +import com.kingsrook.qqq.backend.core.utils.aggregates.InstantAggregates; import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates; +import com.kingsrook.qqq.backend.core.utils.aggregates.LocalDateAggregates; import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates; +import com.kingsrook.qqq.backend.core.utils.aggregates.StringAggregates; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -88,7 +103,7 @@ import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates; ** - summaries (like pivot tables, but called summary to avoid confusion with "native" pivot tables), ** - native pivot tables (not initially supported, due to lack of support in fastexcel...). *******************************************************************************/ -public class GenerateReportAction +public class GenerateReportAction extends AbstractQActionFunction { private static final QLogger LOG = QLogger.getLogger(GenerateReportAction.class); @@ -102,50 +117,67 @@ public class GenerateReportAction // Aggregates: (count:47;sum:10,000;max:2,000;min:15) // // salesSummaryReport > [(state:MO),(city:St.Louis)] > salePrice > (count:47;sum:10,000;max:2,000;min:15) // ///////////////////////////////////////////////////////////////////////////////////////////////////////////// - Map>>> summaryAggregates = new HashMap<>(); - Map>>> varianceAggregates = new HashMap<>(); + Map>>> summaryAggregates = new HashMap<>(); + Map>>> varianceAggregates = new HashMap<>(); - Map> totalAggregates = new HashMap<>(); - Map> varianceTotalAggregates = new HashMap<>(); + Map> totalAggregates = new HashMap<>(); + Map> varianceTotalAggregates = new HashMap<>(); - private QReportMetaData report; - private ReportFormat reportFormat; private ExportStreamerInterface reportStreamer; + private List dataSources; + private List views; + + private Map countByDataSource = new HashMap<>(); /******************************************************************************* ** *******************************************************************************/ - public void execute(ReportInput reportInput) throws QException + public ReportOutput execute(ReportInput reportInput) throws QException { - report = reportInput.getInstance().getReport(reportInput.getReportName()); - reportFormat = reportInput.getReportFormat(); + ReportOutput reportOutput = new ReportOutput(); + QReportMetaData report = getReportMetaData(reportInput); + + this.views = report.getViews(); + this.dataSources = report.getDataSources(); + + ReportFormat reportFormat = reportInput.getReportDestination().getReportFormat(); if(reportFormat == null) { throw new QException("Report format was not specified."); } - reportStreamer = reportFormat.newReportStreamer(); + + if(reportInput.getOverrideExportStreamerSupplier() != null) + { + reportStreamer = reportInput.getOverrideExportStreamerSupplier().get(); + } + else + { + reportStreamer = reportFormat.newReportStreamer(); + } + + reportStreamer.preRun(reportInput.getReportDestination(), views); //////////////////////////////////////////////////////////////////////////////////////////////// // foreach data source, do a query (possibly more than 1, if it goes to multiple table views) // //////////////////////////////////////////////////////////////////////////////////////////////// - for(QReportDataSource dataSource : report.getDataSources()) + for(QReportDataSource dataSource : dataSources) { ////////////////////////////////////////////////////////////////////////////// // make a list of the views that use this data source for various purposes. // ////////////////////////////////////////////////////////////////////////////// - List dataSourceTableViews = report.getViews().stream() + List dataSourceTableViews = views.stream() .filter(v -> v.getType().equals(ReportType.TABLE)) .filter(v -> v.getDataSourceName().equals(dataSource.getName())) .toList(); - List dataSourceSummaryViews = report.getViews().stream() + List dataSourceSummaryViews = views.stream() .filter(v -> v.getType().equals(ReportType.SUMMARY)) .filter(v -> v.getDataSourceName().equals(dataSource.getName())) .toList(); - List dataSourceVariantViews = report.getViews().stream() + List dataSourceVariantViews = views.stream() .filter(v -> v.getType().equals(ReportType.SUMMARY)) .filter(v -> v.getVarianceDataSourceName() != null && v.getVarianceDataSourceName().equals(dataSource.getName())) .toList(); @@ -184,24 +216,50 @@ public class GenerateReportAction //////////////////////////////////////////////////////////////////////////////////// // start the table-view (e.g., open this tab in xlsx) and then run the query-loop // //////////////////////////////////////////////////////////////////////////////////// - startTableView(reportInput, dataSource, dataSourceTableView); + startTableView(reportInput, dataSource, dataSourceTableView, reportFormat); gatherData(reportInput, dataSource, dataSourceTableView, dataSourceSummaryViews, dataSourceVariantViews); } } } + ////////////////////// + // add pivot sheets // + ////////////////////// + for(QReportView view : views) + { + if(view.getType().equals(ReportType.PIVOT)) + { + if(reportFormat.getSupportsNativePivotTables()) + { + startTableView(reportInput, null, view, reportFormat); + } + else + { + LOG.warn("Request to render a report with a PIVOT type view, for a format that does not support native pivot tables", logPair("reportFormat", reportFormat)); + } + + ////////////////////////////////////////////////////////////////////////// + // there's no data to add to a pivot table, so nothing else to do here. // + ////////////////////////////////////////////////////////////////////////// + } + } + outputSummaries(reportInput); + reportOutput.setTotalRecordCount(countByDataSource.values().stream().mapToInt(Integer::intValue).sum()); + reportStreamer.finish(); try { - reportInput.getReportOutputStream().close(); + reportInput.getReportDestination().getReportOutputStream().close(); } catch(Exception e) { throw (new QReportingException("Error completing report", e)); } + + return (reportOutput); } @@ -209,28 +267,48 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView) throws QException + private QReportMetaData getReportMetaData(ReportInput reportInput) throws QException { - QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable()); + if(reportInput.getReportMetaData() != null) + { + return reportInput.getReportMetaData(); + } + if(StringUtils.hasContent(reportInput.getReportName())) + { + return QContext.getQInstance().getReport(reportInput.getReportName()); + } + + throw (new QReportingException("ReportInput did not contain required parameters to identify the report being generated")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView, ReportFormat reportFormat) throws QException + { QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter(); variableInterpreter.addValueMap("input", reportInput.getInputValues()); ExportInput exportInput = new ExportInput(); - exportInput.setReportFormat(reportFormat); - exportInput.setFilename(reportInput.getFilename()); + exportInput.setReportDestination(reportInput.getReportDestination()); exportInput.setTitleRow(getTitle(reportView, variableInterpreter)); exportInput.setIncludeHeaderRow(reportView.getIncludeHeaderRow()); - exportInput.setReportOutputStream(reportInput.getReportOutputStream()); JoinsContext joinsContext = null; - if(StringUtils.hasContent(dataSource.getSourceTable())) + if(dataSource != null) { - joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter()); + if(StringUtils.hasContent(dataSource.getSourceTable())) + { + joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter()); + countDataSourceRecords(reportInput, dataSource, reportFormat); + } } List fields = new ArrayList<>(); - for(QReportField column : reportView.getColumns()) + for(QReportField column : CollectionUtils.nonNullList(reportView.getColumns())) { if(column.getIsVirtual()) { @@ -242,7 +320,7 @@ public class GenerateReportAction JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext == null ? null : joinsContext.getFieldAndTableNameOrAlias(effectiveFieldName); if(fieldAndTableNameOrAlias == null || fieldAndTableNameOrAlias.field() == null) { - throw new QReportingException("Could not find field named [" + effectiveFieldName + "] in dataSource [" + dataSource.getName() + "]"); + throw new QReportingException("Could not find field named [" + effectiveFieldName + "] in dataSource [" + (dataSource == null ? null : dataSource.getName()) + "]"); } QFieldMetaData field = fieldAndTableNameOrAlias.field().clone(); @@ -256,7 +334,7 @@ public class GenerateReportAction } reportStreamer.setDisplayFormats(getDisplayFormatMap(fields)); - reportStreamer.start(exportInput, fields, reportView.getLabel()); + reportStreamer.start(exportInput, fields, reportView.getLabel(), reportView); } @@ -264,7 +342,36 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List summaryViews, List variantViews) throws QException + private void countDataSourceRecords(ReportInput reportInput, QReportDataSource dataSource, ReportFormat reportFormat) throws QException + { + QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone(); + setInputValuesInQueryFilter(reportInput, queryFilter); + + CountInput countInput = new CountInput(); + countInput.setTableName(dataSource.getSourceTable()); + countInput.setFilter(queryFilter); + countInput.setQueryJoins(dataSource.getQueryJoins()); + CountOutput countOutput = new CountAction().execute(countInput); + + if(countOutput.getCount() != null) + { + countByDataSource.put(dataSource.getName(), countOutput.getCount()); + + if(reportFormat.getMaxRows() != null && countOutput.getCount() > reportFormat.getMaxRows()) + { + throw (new QUserFacingException("The requested report would include more rows (" + + String.format("%,d", countOutput.getCount()) + ") than the maximum allowed (" + + String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ").")); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Integer gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List summaryViews, List variantViews) throws QException { //////////////////////////////////////////////////////////////////////////////////////// // check if this view has a transform step - if so, set it up now and run its pre-run // @@ -291,6 +398,9 @@ public class GenerateReportAction RunBackendStepInput finalTransformStepInput = transformStepInput; RunBackendStepOutput finalTransformStepOutput = transformStepOutput; + String tableLabel = ObjectUtils.tryElse(() -> QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel(), Objects.requireNonNullElse(dataSource.getSourceTable(), "")); + AtomicInteger consumedCount = new AtomicInteger(0); + ///////////////////////////////////////////////////////////////// // run a record pipe loop, over the query for this data source // ///////////////////////////////////////////////////////////////// @@ -307,6 +417,7 @@ public class GenerateReportAction queryInput.setTableName(dataSource.getSourceTable()); queryInput.setFilter(queryFilter); queryInput.setQueryJoins(dataSource.getQueryJoins()); + queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS); queryInput.setShouldTranslatePossibleValues(true); queryInput.setFieldsToTranslatePossibleValues(setupFieldsToTranslatePossibleValues(reportInput, dataSource, new JoinsContext(reportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), queryInput.getFilter()))); @@ -350,6 +461,17 @@ public class GenerateReportAction records = finalTransformStepOutput.getRecords(); } + Integer total = countByDataSource.get(dataSource.getName()); + if(total != null) + { + reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records", consumedCount.get() + 1, total); + } + else + { + reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records (" + String.format("%,d", consumedCount.get() + 1) + ")"); + } + consumedCount.getAndAdd(records.size()); + return (consumeRecords(reportInput, dataSource, records, tableView, summaryViews, variantViews)); }); @@ -360,6 +482,8 @@ public class GenerateReportAction { transformStep.postRun(new BackendStepPostRunInput(transformStepInput), new BackendStepPostRunOutput(transformStepOutput)); } + + return consumedCount.get(); } @@ -367,11 +491,11 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private Set setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) + private Set setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) throws QException { Set fieldsToTranslatePossibleValues = new HashSet<>(); - for(QReportView view : report.getViews()) + for(QReportView view : views) { for(QReportField column : CollectionUtils.nonNullList(view.getColumns())) { @@ -385,15 +509,16 @@ public class GenerateReportAction } } - for(String summaryField : CollectionUtils.nonNullList(view.getPivotFields())) + for(String summaryFieldName : CollectionUtils.nonNullList(view.getSummaryFields())) { /////////////////////////////////////////////////////////////////////////////// // all pivotFields that are possible value sources are implicitly translated // /////////////////////////////////////////////////////////////////////////////// - QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable()); - if(table.getField(summaryField).getPossibleValueSourceName() != null) + QTableMetaData mainTable = QContext.getQInstance().getTable(dataSource.getSourceTable()); + FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(mainTable, summaryFieldName); + if(fieldAndJoinTable.field().getPossibleValueSourceName() != null) { - fieldsToTranslatePossibleValues.add(summaryField); + fieldsToTranslatePossibleValues.add(summaryFieldName); } } } @@ -403,6 +528,32 @@ public class GenerateReportAction + /******************************************************************************* + ** + *******************************************************************************/ + public static FieldAndJoinTable getFieldAndJoinTable(QTableMetaData mainTable, String fieldName) throws QException + { + if(fieldName.indexOf('.') > -1) + { + String joinTableName = fieldName.replaceAll("\\..*", ""); + String joinFieldName = fieldName.replaceAll(".*\\.", ""); + + QTableMetaData joinTable = QContext.getQInstance().getTable(joinTableName); + if(joinTable == null) + { + throw (new QException("Unrecognized join table name: " + joinTableName)); + } + + return new FieldAndJoinTable(joinTable.getField(joinFieldName), joinTable); + } + else + { + return new FieldAndJoinTable(mainTable.getField(fieldName), mainTable); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -495,26 +646,27 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List records, Map>>> aggregatesMap) + private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List records, Map>>> aggregatesMap) throws QException { - Map>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>()); + Map>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>()); for(QRecord record : records) { SummaryKey key = new SummaryKey(); - for(String summaryField : view.getPivotFields()) + for(String summaryFieldName : view.getSummaryFields()) { - Serializable summaryValue = record.getValue(summaryField); - if(table.getField(summaryField).getPossibleValueSourceName() != null) + FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName); + Serializable summaryValue = record.getValue(summaryFieldName); + if(fieldAndJoinTable.field().getPossibleValueSourceName() != null) { ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // so, this is kinda a thing - where we implicitly use possible-value labels (e.g., display values) for pivot fields... // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - summaryValue = record.getDisplayValue(summaryField); + summaryValue = record.getDisplayValue(summaryFieldName); } - key.add(summaryField, summaryValue); + key.add(summaryFieldName, summaryValue); - if(view.getIncludePivotSubTotals() && key.getKeys().size() < view.getPivotFields().size()) + if(view.getIncludeSummarySubTotals() && key.getKeys().size() < view.getSummaryFields().size()) { ///////////////////////////////////////////////////////////////////////////////////////// // be careful here, with these key objects, and their identity, being used as map keys // @@ -533,9 +685,9 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map>> viewAggregates, SummaryKey key) + private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map>> viewAggregates, SummaryKey key) throws QException { - Map> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>()); + Map> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>()); addRecordToAggregatesMap(table, record, keyAggregates); } @@ -544,29 +696,74 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map> aggregatesMap) + private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map> aggregatesMap) throws QException { - for(QFieldMetaData field : table.getFields().values()) + ////////////////////////////////////////////////////////////////////////////////////// + // todo - an optimization could be, to only compute aggregates that we'll need... // + // Only if we measure and see this to be slow - it may be, lots of BigDecimal math? // + ////////////////////////////////////////////////////////////////////////////////////// + for(String fieldName : record.getValues().keySet()) { - if(field.getType().equals(QFieldType.INTEGER)) + QFieldMetaData field = null; + try + { + ////////////////////////////////////////////////////// + // todo - memoize this, if we ever need to optimize // + ////////////////////////////////////////////////////// + FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, fieldName); + field = fieldAndJoinTable.field(); + } + catch(Exception e) + { + ////////////////////////////////////////////////////////////////////////////////////// + // for non-real-fields... let's skip for now - but maybe treat as string in future? // + ////////////////////////////////////////////////////////////////////////////////////// + LOG.debug("Couldn't find field in table qInstance - won't compute aggregates for it", logPair("fieldName", fieldName)); + continue; + } + + if(StringUtils.hasContent(field.getPossibleValueSourceName())) { @SuppressWarnings("unchecked") - AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates()); - fieldAggregates.add(record.getValueInteger(field.getName())); + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates()); + fieldAggregates.add(record.getDisplayValue(fieldName)); + } + else if(field.getType().equals(QFieldType.INTEGER)) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new IntegerAggregates()); + fieldAggregates.add(record.getValueInteger(fieldName)); } else if(field.getType().equals(QFieldType.LONG)) { @SuppressWarnings("unchecked") - AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new LongAggregates()); - fieldAggregates.add(record.getValueLong(field.getName())); + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LongAggregates()); + fieldAggregates.add(record.getValueLong(fieldName)); } else if(field.getType().equals(QFieldType.DECIMAL)) { @SuppressWarnings("unchecked") - AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new BigDecimalAggregates()); - fieldAggregates.add(record.getValueBigDecimal(field.getName())); + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new BigDecimalAggregates()); + fieldAggregates.add(record.getValueBigDecimal(fieldName)); + } + else if(field.getType().equals(QFieldType.DATE_TIME)) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new InstantAggregates()); + fieldAggregates.add(record.getValueInstant(fieldName)); + } + else if(field.getType().equals(QFieldType.DATE)) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LocalDateAggregates()); + fieldAggregates.add(record.getValueLocalDate(fieldName)); + } + if(field.getType().isStringLike()) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates()); + fieldAggregates.add(record.getValueString(fieldName)); } - // todo - more types (dates, at least?) } } @@ -575,24 +772,22 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private void outputSummaries(ReportInput reportInput) throws QReportingException, QFormulaException + private void outputSummaries(ReportInput reportInput) throws QException { - List reportViews = report.getViews().stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList(); + List reportViews = views.stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList(); for(QReportView view : reportViews) { - QReportDataSource dataSource = report.getDataSource(view.getDataSourceName()); + QReportDataSource dataSource = getDataSource(view.getDataSourceName()); QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable()); SummaryOutput summaryOutput = computeSummaryRowsForView(reportInput, view, table); ExportInput exportInput = new ExportInput(); - exportInput.setReportFormat(reportFormat); - exportInput.setFilename(reportInput.getFilename()); + exportInput.setReportDestination(reportInput.getReportDestination()); exportInput.setTitleRow(summaryOutput.titleRow); exportInput.setIncludeHeaderRow(view.getIncludeHeaderRow()); - exportInput.setReportOutputStream(reportInput.getReportOutputStream()); reportStreamer.setDisplayFormats(getDisplayFormatMap(view)); - reportStreamer.start(exportInput, getFields(table, view), view.getLabel()); + reportStreamer.start(exportInput, getFields(table, view), view.getLabel(), view); reportStreamer.addRecords(summaryOutput.summaryRows); // todo - what if this set is huge? @@ -605,6 +800,24 @@ public class GenerateReportAction + /******************************************************************************* + ** + *******************************************************************************/ + private QReportDataSource getDataSource(String dataSourceName) + { + for(QReportDataSource dataSource : CollectionUtils.nonNullList(dataSources)) + { + if(dataSource.getName().equals(dataSourceName)) + { + return (dataSource); + } + } + + return (null); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -632,13 +845,13 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private List getFields(QTableMetaData table, QReportView view) + private List getFields(QTableMetaData table, QReportView view) throws QException { List fields = new ArrayList<>(); - for(String pivotField : view.getPivotFields()) + for(String summaryFieldName : view.getSummaryFields()) { - QFieldMetaData field = table.getField(pivotField); - fields.add(new QFieldMetaData(pivotField, field.getType()).withLabel(field.getLabel())); // todo do we need the type? if so need table as input here + FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName); + fields.add(new QFieldMetaData(summaryFieldName, fieldAndJoinTable.field().getType()).withLabel(fieldAndJoinTable.field().getLabel())); // todo do we need the type? if so need table as input here } for(QReportField column : view.getColumns()) { @@ -668,11 +881,11 @@ public class GenerateReportAction // create summary rows // ///////////////////////// List summaryRows = new ArrayList<>(); - for(Map.Entry>> entry : summaryAggregates.getOrDefault(view.getName(), Collections.emptyMap()).entrySet()) + for(Map.Entry>> entry : summaryAggregates.getOrDefault(view.getName(), Collections.emptyMap()).entrySet()) { - SummaryKey summaryKey = entry.getKey(); - Map> fieldAggregates = entry.getValue(); - Map summaryValues = getSummaryValuesForInterpreter(fieldAggregates); + SummaryKey summaryKey = entry.getKey(); + Map> fieldAggregates = entry.getValue(); + Map summaryValues = getSummaryValuesForInterpreter(fieldAggregates); variableInterpreter.addValueMap("pivot", summaryValues); variableInterpreter.addValueMap("summary", summaryValues); @@ -681,9 +894,9 @@ public class GenerateReportAction if(!varianceAggregates.isEmpty()) { - Map>> varianceMap = varianceAggregates.getOrDefault(view.getName(), Collections.emptyMap()); - Map> varianceSubMap = varianceMap.getOrDefault(summaryKey, Collections.emptyMap()); - Map varianceValues = getSummaryValuesForInterpreter(varianceSubMap); + Map>> varianceMap = varianceAggregates.getOrDefault(view.getName(), Collections.emptyMap()); + Map> varianceSubMap = varianceMap.getOrDefault(summaryKey, Collections.emptyMap()); + Map varianceValues = getSummaryValuesForInterpreter(varianceSubMap); variableInterpreter.addValueMap("variancePivot", varianceValues); variableInterpreter.addValueMap("variance", varianceValues); } @@ -702,7 +915,7 @@ public class GenerateReportAction /////////////////////////////////////////////////////////////////////////////// // for summary subtotals, add the text "Total" to the last field in this key // /////////////////////////////////////////////////////////////////////////////// - if(summaryKey.getKeys().size() < view.getPivotFields().size()) + if(summaryKey.getKeys().size() < view.getSummaryFields().size()) { String fieldName = summaryKey.getKeys().get(summaryKey.getKeys().size() - 1).getA(); summaryRow.setValue(fieldName, summaryRow.getValueString(fieldName) + " Total"); @@ -740,11 +953,11 @@ public class GenerateReportAction { totalRow = new QRecord(); - for(String pivotField : view.getPivotFields()) + for(String summaryField : view.getSummaryFields()) { if(totalRow.getValues().isEmpty()) { - totalRow.setValue(pivotField, "Totals"); + totalRow.setValue(summaryField, "Totals"); } } @@ -864,18 +1077,24 @@ public class GenerateReportAction /******************************************************************************* ** *******************************************************************************/ - private Map getSummaryValuesForInterpreter(Map> fieldAggregates) + private Map getSummaryValuesForInterpreter(Map> fieldAggregates) { Map summaryValuesForInterpreter = new HashMap<>(); - for(Map.Entry> subEntry : fieldAggregates.entrySet()) + for(Map.Entry> subEntry : fieldAggregates.entrySet()) { - String fieldName = subEntry.getKey(); - AggregatesInterface aggregates = subEntry.getValue(); + String fieldName = subEntry.getKey(); + AggregatesInterface aggregates = subEntry.getValue(); summaryValuesForInterpreter.put("sum." + fieldName, aggregates.getSum()); summaryValuesForInterpreter.put("count." + fieldName, aggregates.getCount()); + summaryValuesForInterpreter.put("count_nums." + fieldName, aggregates.getCount()); summaryValuesForInterpreter.put("min." + fieldName, aggregates.getMin()); summaryValuesForInterpreter.put("max." + fieldName, aggregates.getMax()); summaryValuesForInterpreter.put("average." + fieldName, aggregates.getAverage()); + summaryValuesForInterpreter.put("product." + fieldName, aggregates.getProduct()); + summaryValuesForInterpreter.put("var." + fieldName, aggregates.getVariance()); + summaryValuesForInterpreter.put("varp." + fieldName, aggregates.getVarP()); + summaryValuesForInterpreter.put("std_dev." + fieldName, aggregates.getStandardDeviation()); + summaryValuesForInterpreter.put("std_devp." + fieldName, aggregates.getStdDevP()); } return summaryValuesForInterpreter; } @@ -889,4 +1108,27 @@ public class GenerateReportAction { } + + + /******************************************************************************* + ** + *******************************************************************************/ + public record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable) + { + + /******************************************************************************* + ** + *******************************************************************************/ + public String getLabel(QTableMetaData mainTable) + { + if(mainTable.getName().equals(joinTable.getName())) + { + return (field.getLabel()); + } + else + { + return (joinTable.getLabel() + ": " + field.getLabel()); + } + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java index b90cde16..0b798fe8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java @@ -26,17 +26,23 @@ import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.JsonUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; /******************************************************************************* @@ -46,13 +52,23 @@ public class JsonExportStreamer implements ExportStreamerInterface { private static final QLogger LOG = QLogger.getLogger(JsonExportStreamer.class); + private boolean prettyPrint = true; + private ExportInput exportInput; private QTableMetaData table; private List fields; private OutputStream outputStream; - private boolean needComma = false; - private boolean prettyPrint = true; + private boolean multipleViews = false; + private boolean haveStartedAnyViews = false; + + private boolean needCommaBeforeRecord = false; + + private byte[] indent = new byte[0]; + private String indentString = ""; + + private Pattern colonLetterPattern = Pattern.compile(":([A-Z]+)($|[A-Z][a-z])"); + private Memoization fieldLabelMemoization = new Memoization<>(); @@ -69,21 +85,124 @@ public class JsonExportStreamer implements ExportStreamerInterface ** *******************************************************************************/ @Override - public void start(ExportInput exportInput, List fields, String label) throws QReportingException + public void preRun(ReportDestination reportDestination, List views) throws QReportingException + { + outputStream = reportDestination.getReportOutputStream(); + + if(views.size() > 1) + { + multipleViews = true; + } + + if(multipleViews) + { + try + { + indentIfPretty(outputStream); + outputStream.write('['); + newlineIfPretty(outputStream); + increaseIndent(); + } + catch(IOException e) + { + throw (new QReportingException("Error starting report output", e)); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void start(ExportInput exportInput, List fields, String label, QReportView view) throws QReportingException { this.exportInput = exportInput; this.fields = fields; table = exportInput.getTable(); - outputStream = this.exportInput.getReportOutputStream(); + + needCommaBeforeRecord = false; try { + if(multipleViews) + { + if(haveStartedAnyViews) + { + ///////////////////////// + // close the last view // + ///////////////////////// + newlineIfPretty(outputStream); + + decreaseIndent(); + indentIfPretty(outputStream); + outputStream.write(']'); + newlineIfPretty(outputStream); + + decreaseIndent(); + indentIfPretty(outputStream); + outputStream.write('}'); + outputStream.write(','); + newlineIfPretty(outputStream); + } + + ///////////////////////////////////////////////////////////// + // open a new view, as an object, with a name & data entry // + ///////////////////////////////////////////////////////////// + indentIfPretty(outputStream); + outputStream.write('{'); + newlineIfPretty(outputStream); + increaseIndent(); + + indentIfPretty(outputStream); + outputStream.write(String.format(""" + "name":"%s",""", label).getBytes(StandardCharsets.UTF_8)); + newlineIfPretty(outputStream); + + indentIfPretty(outputStream); + outputStream.write(""" + "data":""".getBytes(StandardCharsets.UTF_8)); + newlineIfPretty(outputStream); + } + + ////////////////////////////////////////////// + // start the array of entries for this view // + ////////////////////////////////////////////// + indentIfPretty(outputStream); outputStream.write('['); + increaseIndent(); } catch(IOException e) { throw (new QReportingException("Error starting report output", e)); } + + haveStartedAnyViews = true; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void increaseIndent() + { + indent = new byte[indent.length + 3]; + Arrays.fill(indent, (byte) ' '); + indentString = new String(indent); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void decreaseIndent() + { + indent = new byte[Math.max(0, indent.length - 3)]; + Arrays.fill(indent, (byte) ' '); + indentString = new String(indent); } @@ -111,7 +230,7 @@ public class JsonExportStreamer implements ExportStreamerInterface { try { - if(needComma) + if(needCommaBeforeRecord) { outputStream.write(','); } @@ -119,21 +238,25 @@ public class JsonExportStreamer implements ExportStreamerInterface Map mapForJson = new LinkedHashMap<>(); for(QFieldMetaData field : fields) { - String labelForJson = StringUtils.lcFirst(field.getLabel().replace(" ", "")); - mapForJson.put(labelForJson, qRecord.getValue(field.getName())); + mapForJson.put(getLabelForJson(field), qRecord.getValue(field.getName())); } String json = prettyPrint ? JsonUtils.toPrettyJson(mapForJson) : JsonUtils.toJson(mapForJson); + if(prettyPrint) + { + json = json.replaceAll("(?s)\n", "\n" + indentString); + } if(prettyPrint) { outputStream.write('\n'); } + indentIfPretty(outputStream); outputStream.write(json.getBytes(StandardCharsets.UTF_8)); outputStream.flush(); // todo - less often? - needComma = true; + needCommaBeforeRecord = true; } catch(Exception e) { @@ -143,6 +266,73 @@ public class JsonExportStreamer implements ExportStreamerInterface + /******************************************************************************* + ** + *******************************************************************************/ + String getLabelForJson(QFieldMetaData field) + { + ////////////////////////////////////////////////////////////////////////// + // memoize, to avoid running these regex/replacements millions of times // + ////////////////////////////////////////////////////////////////////////// + Optional result = fieldLabelMemoization.getResult(field.getName(), fieldName -> + { + String labelForJson = field.getLabel().replace(" ", ""); + + ///////////////////////////////////////////////////////////////////////////// + // now fix up any field-name-parts after the table: portion of a join name // + // lineItem:SKU to become lineItem:sku // + // parcel:SLAStatus to become parcel:slaStatus // + // order:Client to become order:client // + ///////////////////////////////////////////////////////////////////////////// + Matcher allCaps = Pattern.compile("^[A-Z]+$").matcher(labelForJson); + Matcher startsAllCapsThenNextWordMatcher = Pattern.compile("([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson); + Matcher startsOneCapMatcher = Pattern.compile("([A-Z])(.*)").matcher(labelForJson); + + if(allCaps.matches()) + { + labelForJson = allCaps.replaceAll(m -> m.group().toLowerCase()); + } + else if(startsAllCapsThenNextWordMatcher.matches()) + { + labelForJson = startsAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2)); + } + else if(startsOneCapMatcher.matches()) + { + labelForJson = startsOneCapMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2)); + } + + ///////////////////////////////////////////////////////////////////////////// + // now fix up any field-name-parts after the table: portion of a join name // + // lineItem:SKU to become lineItem:sku // + // parcel:SLAStatus to become parcel:slaStatus // + // order:Client to become order:client // + ///////////////////////////////////////////////////////////////////////////// + Matcher colonThenAllCapsThenEndMatcher = Pattern.compile("(.*:)([A-Z]+)$").matcher(labelForJson); + Matcher colonThenAllCapsThenNextWordMatcher = Pattern.compile("(.*:)([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson); + Matcher colonThenOneCapMatcher = Pattern.compile("(.*:)([A-Z])(.*)").matcher(labelForJson); + + if(colonThenAllCapsThenEndMatcher.matches()) + { + labelForJson = colonThenAllCapsThenEndMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase()); + } + else if(colonThenAllCapsThenNextWordMatcher.matches()) + { + labelForJson = colonThenAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3)); + } + else if(colonThenOneCapMatcher.matches()) + { + labelForJson = colonThenOneCapMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3)); + } + + System.out.println("Label: " + labelForJson); + return (labelForJson); + }); + + return result.orElse(field.getName()); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -162,11 +352,34 @@ public class JsonExportStreamer implements ExportStreamerInterface { try { - if(prettyPrint) - { - outputStream.write('\n'); - } + ////////////////////////////////////////////// + // close the array of entries for this view // + ////////////////////////////////////////////// + newlineIfPretty(outputStream); + + decreaseIndent(); + indentIfPretty(outputStream); outputStream.write(']'); + newlineIfPretty(outputStream); + + if(multipleViews) + { + //////////////////////////////////////////// + // close this view, if there are multiple // + //////////////////////////////////////////// + decreaseIndent(); + indentIfPretty(outputStream); + outputStream.write('}'); + newlineIfPretty(outputStream); + + ///////////////////////////// + // close the list of views // + ///////////////////////////// + decreaseIndent(); + indentIfPretty(outputStream); + outputStream.write(']'); + newlineIfPretty(outputStream); + } } catch(IOException e) { @@ -174,4 +387,30 @@ public class JsonExportStreamer implements ExportStreamerInterface } } + + + /******************************************************************************* + ** + *******************************************************************************/ + private void newlineIfPretty(OutputStream outputStream) throws IOException + { + if(prettyPrint) + { + outputStream.write('\n'); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void indentIfPretty(OutputStream outputStream) throws IOException + { + if(prettyPrint) + { + outputStream.write(indent); + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ListOfMapsExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ListOfMapsExportStreamer.java index 365c3a40..7e422a21 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ListOfMapsExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ListOfMapsExportStreamer.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; /******************************************************************************* @@ -87,7 +88,7 @@ public class ListOfMapsExportStreamer implements ExportStreamerInterface ** *******************************************************************************/ @Override - public void start(ExportInput exportInput, List fields, String label) throws QReportingException + public void start(ExportInput exportInput, List fields, String label, QReportView view) throws QReportingException { this.exportInput = exportInput; this.fields = fields; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportUtils.java new file mode 100644 index 00000000..9c682e22 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportUtils.java @@ -0,0 +1,51 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.exceptions.QReportingException; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ReportUtils +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public static QReportView getSourceViewForPivotTableView(List views, QReportView pivotTableView) throws QReportingException + { + Optional sourceView = views.stream().filter(v -> v.getName().equals(pivotTableView.getPivotTableSourceViewName())).findFirst(); + if(sourceView.isEmpty()) + { + throw (new QReportingException("Could not find data view [" + pivotTableView.getPivotTableSourceViewName() + "] for pivot table view [" + pivotTableView.getName() + "]")); + } + + return sourceView.get(); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/BoldHeaderAndFooterExcelStyler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/BoldHeaderAndFooterFastExcelStyler.java similarity index 85% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/BoldHeaderAndFooterExcelStyler.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/BoldHeaderAndFooterFastExcelStyler.java index 12dc6685..45d37640 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/BoldHeaderAndFooterExcelStyler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/BoldHeaderAndFooterFastExcelStyler.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting; +package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel; import org.dhatim.fastexcel.BorderSide; @@ -30,7 +30,7 @@ import org.dhatim.fastexcel.StyleSetter; /******************************************************************************* ** Version of excel styler that does bold headers and footers, with basic borders. *******************************************************************************/ -public class BoldHeaderAndFooterExcelStyler implements ExcelStylerInterface +public class BoldHeaderAndFooterFastExcelStyler implements FastExcelStylerInterface { /******************************************************************************* @@ -60,6 +60,9 @@ public class BoldHeaderAndFooterExcelStyler implements ExcelStylerInterface + /******************************************************************************* + ** + *******************************************************************************/ @Override public void styleTotalsRow(StyleSetter totalsRowStyle) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/ExcelFastexcelExportStreamer.java similarity index 89% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelExportStreamer.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/ExcelFastexcelExportStreamer.java index 61f14588..fd1dc915 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/ExcelFastexcelExportStreamer.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.actions.reporting; +package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel; import java.io.OutputStream; @@ -34,8 +34,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import com.kingsrook.qqq.backend.core.actions.reporting.excelformatting.ExcelStylerInterface; -import com.kingsrook.qqq.backend.core.actions.reporting.excelformatting.PlainExcelStyler; +import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QReportingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; @@ -43,7 +43,9 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.dhatim.fastexcel.StyleSetter; @@ -52,19 +54,19 @@ import org.dhatim.fastexcel.Worksheet; /******************************************************************************* - ** Excel export format implementation + ** Excel export format implementation - built using fastexcel library *******************************************************************************/ -public class ExcelExportStreamer implements ExportStreamerInterface +public class ExcelFastexcelExportStreamer implements ExportStreamerInterface { - private static final QLogger LOG = QLogger.getLogger(ExcelExportStreamer.class); + private static final QLogger LOG = QLogger.getLogger(ExcelFastexcelExportStreamer.class); private ExportInput exportInput; private QTableMetaData table; private List fields; private OutputStream outputStream; - private ExcelStylerInterface excelStylerInterface = new PlainExcelStyler(); - private Map excelCellFormats; + private FastExcelStylerInterface fastExcelStylerInterface = new PlainFastExcelStyler(); + private Map excelCellFormats; private Workbook workbook; private Worksheet worksheet; @@ -76,7 +78,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface /******************************************************************************* ** *******************************************************************************/ - public ExcelExportStreamer() + public ExcelFastexcelExportStreamer() { } @@ -105,14 +107,14 @@ public class ExcelExportStreamer implements ExportStreamerInterface ** Starts a new worksheet in the current workbook. Can be called multiple times. *******************************************************************************/ @Override - public void start(ExportInput exportInput, List fields, String label) throws QReportingException + public void start(ExportInput exportInput, List fields, String label, QReportView view) throws QReportingException { try { this.exportInput = exportInput; this.fields = fields; table = exportInput.getTable(); - outputStream = this.exportInput.getReportOutputStream(); + outputStream = this.exportInput.getReportDestination().getReportOutputStream(); this.row = 0; this.sheetCount++; @@ -121,7 +123,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface ///////////////////////////////////////////////////////////////////////////////////////////////////// if(workbook == null) { - String appName = "QQQ"; + String appName = ObjectUtils.tryAndRequireNonNullElse(() -> QContext.getQInstance().getBranding().getAppName(), "QQQ"); QInstance instance = exportInput.getInstance(); if(instance != null && instance.getBranding() != null && instance.getBranding().getCompanyName() != null) { @@ -167,7 +169,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface worksheet.range(row, 0, row, fields.size() - 1).merge(); StyleSetter titleStyle = worksheet.range(row, 0, row, fields.size() - 1).style(); - excelStylerInterface.styleTitleRow(titleStyle); + fastExcelStylerInterface.styleTitleRow(titleStyle); titleStyle.set(); row++; @@ -187,7 +189,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface } StyleSetter headerStyle = worksheet.range(row, 0, row, fields.size() - 1).style(); - excelStylerInterface.styleHeaderRow(headerStyle); + fastExcelStylerInterface.styleHeaderRow(headerStyle); headerStyle.set(); row++; @@ -315,7 +317,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface writeRecord(record); StyleSetter totalsRowStyle = worksheet.range(row, 0, row, fields.size() - 1).style(); - excelStylerInterface.styleTotalsRow(totalsRowStyle); + fastExcelStylerInterface.styleTotalsRow(totalsRowStyle); totalsRowStyle.set(); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/ExcelStylerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/FastExcelStylerInterface.java similarity index 92% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/ExcelStylerInterface.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/FastExcelStylerInterface.java index ed58aba3..68b361eb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/ExcelStylerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/FastExcelStylerInterface.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting; +package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel; import org.dhatim.fastexcel.StyleSetter; @@ -29,7 +29,7 @@ import org.dhatim.fastexcel.StyleSetter; ** Interface for classes that know how to apply styles to an Excel stream being ** built by fastexcel. *******************************************************************************/ -public interface ExcelStylerInterface +public interface FastExcelStylerInterface { /******************************************************************************* diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/PlainFastExcelStyler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/PlainFastExcelStyler.java new file mode 100644 index 00000000..e42afe36 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/PlainFastExcelStyler.java @@ -0,0 +1,43 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel; + + +import org.dhatim.fastexcel.StyleSetter; + + +/******************************************************************************* + ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface. + *******************************************************************************/ +public class PlainFastExcelStyler implements FastExcelStylerInterface +{ + + /******************************************************************************* + ** ... sorry, but adding this gives us test coverage on this class, even though + ** we're just deferring to super... + *******************************************************************************/ + @Override + public void styleHeaderRow(StyleSetter headerRowStyle) + { + FastExcelStylerInterface.super.styleHeaderRow(headerRowStyle); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/BoldHeaderAndFooterPoiExcelStyler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/BoldHeaderAndFooterPoiExcelStyler.java new file mode 100644 index 00000000..c30beef1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/BoldHeaderAndFooterPoiExcelStyler.java @@ -0,0 +1,93 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi; + + +import org.apache.poi.ss.usermodel.BorderStyle; +import org.apache.poi.ss.usermodel.CreationHelper; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.HorizontalAlignment; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + + +/******************************************************************************* + ** Version of POI excel styler that does bold headers and footers, with basic borders. + *******************************************************************************/ +public class BoldHeaderAndFooterPoiExcelStyler implements PoiExcelStylerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public XSSFCellStyle createStyleForTitle(XSSFWorkbook workbook, CreationHelper createHelper) + { + Font font = workbook.createFont(); + font.setFontHeightInPoints((short) 14); + font.setBold(true); + + XSSFCellStyle cellStyle = workbook.createCellStyle(); + cellStyle.setFont(font); + cellStyle.setAlignment(HorizontalAlignment.CENTER); + + return (cellStyle); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper) + { + Font font = workbook.createFont(); + font.setBold(true); + + XSSFCellStyle cellStyle = workbook.createCellStyle(); + cellStyle.setFont(font); + cellStyle.setBorderBottom(BorderStyle.THIN); + + return (cellStyle); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public XSSFCellStyle createStyleForFooter(XSSFWorkbook workbook, CreationHelper createHelper) + { + Font font = workbook.createFont(); + font.setBold(true); + + XSSFCellStyle cellStyle = workbook.createCellStyle(); + cellStyle.setFont(font); + cellStyle.setBorderTop(BorderStyle.THIN); + cellStyle.setBorderBottom(BorderStyle.DOUBLE); + + return (cellStyle); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java new file mode 100644 index 00000000..1ce29911 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java @@ -0,0 +1,819 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi; + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Serializable; +import java.io.Writer; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; +import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface; +import com.kingsrook.qqq.backend.core.actions.reporting.ReportUtils; +import com.kingsrook.qqq.backend.core.exceptions.QReportingException; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.poi.ss.SpreadsheetVersion; +import org.apache.poi.ss.usermodel.CreationHelper; +import org.apache.poi.ss.usermodel.DataConsolidateFunction; +import org.apache.poi.ss.usermodel.DateUtil; +import org.apache.poi.ss.util.AreaReference; +import org.apache.poi.ss.util.CellReference; +import org.apache.poi.ss.util.WorkbookUtil; +import org.apache.poi.xssf.usermodel.XSSFCell; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; +import org.apache.poi.xssf.usermodel.XSSFPivotTable; +import org.apache.poi.xssf.usermodel.XSSFRow; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + + +/******************************************************************************* + ** Excel export format implementation using POI library, but with modifications + ** to actually stream output rather than use any temp files. + ** + ** For a rough outline: + ** - create a basically empty Excel workbook using POI - empty meaning, without + ** data rows. + ** - have POI write that workbook out into a byte[] - which will be a zip full + ** of xml (e.g., xlsx). + ** - then open a new ZipOutputStream wrapper around the OutputStream we took in + ** as the report destination (e.g., streamed into storage or http output) + ** - Copy over all entries from the xlsx into our new zip-output-stream, other than + ** ones that are the actual sheets that we want to put data into. + ** - For the sheet entries, use the StreamedPoiSheetWriter class to write the + ** report's data directly out as Excel XML (not using POI). + ** - Pivot tables require a bit of an additional hack - to write a "pivot cache + ** definition", which, while we won't put all the data in it (because we'll tell + ** it refreshOnLoad="true"), we will at least need to know how many cols & rows + ** are in the data-sheet (which we wouldn't know until we streamed that sheet!) + *******************************************************************************/ +public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInterface +{ + private static final QLogger LOG = QLogger.getLogger(ExcelPoiBasedStreamingExportStreamer.class); + + private List views; + private ExportInput exportInput; + private List fields; + private OutputStream outputStream; + private ZipOutputStream zipOutputStream; + + public static final String EXCEL_DATE_FORMAT = "yyyy-MM-dd"; + public static final String EXCEL_DATE_TIME_FORMAT = "yyyy-MM-dd H:mm:ss"; + + private PoiExcelStylerInterface poiExcelStylerInterface = getStylerInterface(); + private Map excelCellFormats; + private Map styles = new HashMap<>(); + + private int rowNo = 0; + private int sheetIndex = 1; + + private Map pivotViewToCacheDefinitionReferenceMap = new HashMap<>(); + + private Writer activeSheetWriter = null; + private StreamedSheetWriter sheetWriter = null; + + private QReportView currentView = null; + private Map> fieldsPerView = new HashMap<>(); + private Map rowsPerView = new HashMap<>(); + private Map labelViewsByName = new HashMap<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ExcelPoiBasedStreamingExportStreamer() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void preRun(ReportDestination reportDestination, List views) throws QReportingException + { + try + { + this.outputStream = reportDestination.getReportOutputStream(); + this.views = views; + + /////////////////////////////////////////////////////////////////////////////// + // create 'template' workbook through poi - with sheets corresponding to our // + // actual file this will be a zip file (stream), with entries for all of the // + // files in the final xlsx but without any data, so it'll be small // + /////////////////////////////////////////////////////////////////////////////// + XSSFWorkbook workbook = new XSSFWorkbook(); + createStyles(workbook); + + ////////////////////////////////////////////////////////////////////////////////////////////////// + // for each of the sheets, create it in the workbook, and put a reference to it in the sheetMap // + ////////////////////////////////////////////////////////////////////////////////////////////////// + Map sheetMapByExcelReference = new HashMap<>(); + Map sheetMapByViewName = new HashMap<>(); + + int sheetCounter = 1; + for(QReportView view : views) + { + String label = Objects.requireNonNullElse(view.getLabel(), "Sheet " + sheetCounter); + label = WorkbookUtil.createSafeSheetName(label); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // track the actually-used sheet labels (needed for referencing in pivot table generation) // + ///////////////////////////////////////////////////////////////////////////////////////////// + labelViewsByName.put(view.getName(), label); + + XSSFSheet sheet = workbook.createSheet(label); + String sheetReference = sheet.getPackagePart().getPartName().getName().substring(1); + sheetMapByExcelReference.put(sheetReference, sheet); + sheetMapByViewName.put(view.getName(), sheet); + sheetCounter++; + } + + //////////////////////////////////////////////////// + // if any views are pivot tables, create them now // + //////////////////////////////////////////////////// + List pivotViewNames = new ArrayList<>(); + for(QReportView view : views) + { + if(ReportType.PIVOT.equals(view.getType())) + { + pivotViewNames.add(view.getName()); + + XSSFSheet pivotTableSheet = Objects.requireNonNull(sheetMapByViewName.get(view.getName()), "Could not get pivot table sheet view by name: " + view.getName()); + XSSFSheet dataSheet = Objects.requireNonNull(sheetMapByViewName.get(view.getPivotTableSourceViewName()), "Could not get pivot table source sheet by view name: " + view.getPivotTableSourceViewName()); + QReportView dataView = ReportUtils.getSourceViewForPivotTableView(views, view); + createPivotTableTemplate(pivotTableSheet, view, dataSheet, dataView); + } + } + Iterator pivotViewNameIterator = pivotViewNames.iterator(); + + ///////////////////////////////////////////////////////// + // write that template worksheet zip out to byte array // + ///////////////////////////////////////////////////////// + ByteArrayOutputStream templateBAOS = new ByteArrayOutputStream(); + workbook.write(templateBAOS); + templateBAOS.close(); + byte[] templateBytes = templateBAOS.toByteArray(); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // open up a zipOutputStream around the output stream that the report is to be written to. // + // note: this stream is closed in the finish method - see more comments there. // + ///////////////////////////////////////////////////////////////////////////////////////////// + this.zipOutputStream = new ZipOutputStream(this.outputStream); + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // copy over all the entries in the template zip that aren't the sheets into the output stream // + ///////////////////////////////////////////////////////////////////////////////////////////////// + ZipInputStream zipInputStream = new ZipInputStream(new ByteArrayInputStream(templateBytes)); + ZipEntry zipTemplateEntry = null; + byte[] buffer = new byte[2048]; + while((zipTemplateEntry = zipInputStream.getNextEntry()) != null) + { + if(zipTemplateEntry.getName().matches(".*/pivotCacheDefinition.*.xml")) + { + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // if this zip entry is a pivotCacheDefinition, then don't write it to the output stream right now. // + // instead, just map the pivot view's name to the zipTemplateEntry name // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + if(!pivotViewNameIterator.hasNext()) + { + throw new QReportingException("Found a pivot cache definition [" + zipTemplateEntry.getName() + "] in the template ZIP, but no (more) corresponding pivot view names"); + } + + String pivotViewName = pivotViewNameIterator.next(); + LOG.info("Holding on a pivot cache definition zip template entry [" + pivotViewName + "] [" + zipTemplateEntry.getName() + "]..."); + pivotViewToCacheDefinitionReferenceMap.put(pivotViewName, zipTemplateEntry.getName()); + } + else if(!sheetMapByExcelReference.containsKey(zipTemplateEntry.getName())) + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else, if we don't have this zipTemplateEntry name in our map of sheets, then this is a kinda "meta" // + // file that we don't really care about (e.g., not our sheet data), so just copy it to the output stream. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + LOG.info("Copying zip template entry [" + zipTemplateEntry.getName() + "] to output stream"); + zipOutputStream.putNextEntry(new ZipEntry(zipTemplateEntry.getName())); + + int length; + while((length = zipInputStream.read(buffer)) > 0) + { + zipOutputStream.write(buffer, 0, length); + } + + zipInputStream.closeEntry(); + } + else + { + //////////////////////////////////////////////////////////////////////////////////// + // else - this is a sheet - so again, don't write it yet - stream its data below. // + //////////////////////////////////////////////////////////////////////////////////// + LOG.info("Skipping presumed sheet zip template entry [" + zipTemplateEntry.getName() + "] to output stream"); + } + } + + zipInputStream.close(); + } + catch(Exception e) + { + throw (new QReportingException("Error preparing to generate spreadsheet", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void createPivotTableTemplate(XSSFSheet pivotTableSheet, QReportView pivotView, XSSFSheet dataSheet, QReportView dataView) throws QReportingException + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // write just enough data to the dataView's sheet so that we can refer to it for creating the pivot table. // + // we need to do this, because POI will try to create the pivotCache referring to the data sheet, and if // + // there isn't any data there, it'll crash. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + XSSFRow headerRow = dataSheet.createRow(0); + int columnNo = 0; + for(QReportField column : dataView.getColumns()) + { + XSSFCell cell = headerRow.createCell(columnNo++); + cell.setCellValue(QInstanceEnricher.nameToLabel(column.getName())); + } + + XSSFRow valuesRow = dataSheet.createRow(1); + columnNo = 0; + for(QReportField column : dataView.getColumns()) + { + XSSFCell cell = valuesRow.createCell(columnNo++); + cell.setCellValue("Value " + columnNo); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // for this template version of the pivot table, tell it there are only 2 rows in the source sheet // + // as that's all that we wrote above (a header and 1 fake value row) // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + int rows = 2; + String colsLetter = CellReference.convertNumToColString(dataView.getColumns().size() - 1); + AreaReference source = new AreaReference("A1:" + colsLetter + rows, SpreadsheetVersion.EXCEL2007); + CellReference position = new CellReference("A1"); + + ////////////////////////////////////////////////////////////////// + // tell poi all about our pivot table - rows, cols, and columns // + ////////////////////////////////////////////////////////////////// + XSSFPivotTable pivotTable = pivotTableSheet.createPivotTable(source, position, dataSheet); + + for(PivotTableGroupBy row : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getRows())) + { + int rowLabelColumnIndex = getColumnIndex(dataView.getColumns(), row.getFieldName()); + pivotTable.addRowLabel(rowLabelColumnIndex); + } + + for(PivotTableGroupBy column : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getColumns())) + { + int colLabelColumnIndex = getColumnIndex(dataView.getColumns(), column.getFieldName()); + pivotTable.addColLabel(colLabelColumnIndex); + } + + for(PivotTableValue value : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getValues())) + { + int columnLabelColumnIndex = getColumnIndex(dataView.getColumns(), value.getFieldName()); + + String labelPrefix = value.getFunction().name() + " of "; + String label = labelPrefix + QInstanceEnricher.nameToLabel(value.getFieldName()); + String valueFormat = null; + + Optional optSourceField = dataView.getColumns().stream().filter(c -> c.getName().equals(value.getFieldName())).findFirst(); + if(optSourceField.isPresent()) + { + QReportField sourceField = optSourceField.get(); + + if(StringUtils.hasContent(sourceField.getLabel())) + { + label = labelPrefix + sourceField.getLabel(); + } + + if(StringUtils.hasContent(sourceField.getDisplayFormat())) + { + valueFormat = DisplayFormat.getExcelFormat(sourceField.getDisplayFormat()); + } + else + { + if(QFieldType.DATE.equals(sourceField.getType())) + { + valueFormat = EXCEL_DATE_FORMAT; + } + else if(QFieldType.DATE_TIME.equals(sourceField.getType())) + { + valueFormat = EXCEL_DATE_TIME_FORMAT; + } + } + } + + pivotTable.addColumnLabel(DataConsolidateFunction.valueOf(value.getFunction().name()), columnLabelColumnIndex, label, valueFormat); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private int getColumnIndex(List columns, String fieldName) throws QReportingException + { + for(int i = 0; i < columns.size(); i++) + { + if(columns.get(i).getName().equals(fieldName)) + { + return (i); + } + } + + throw (new QReportingException("Could not find column by name [" + fieldName + "]")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void createStyles(XSSFWorkbook workbook) + { + CreationHelper createHelper = workbook.getCreationHelper(); + + XSSFCellStyle dateStyle = workbook.createCellStyle(); + dateStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_FORMAT)); + styles.put("date", dateStyle); + + XSSFCellStyle dateTimeStyle = workbook.createCellStyle(); + dateTimeStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_TIME_FORMAT)); + styles.put("datetime", dateTimeStyle); + + styles.put("title", poiExcelStylerInterface.createStyleForTitle(workbook, createHelper)); + styles.put("header", poiExcelStylerInterface.createStyleForHeader(workbook, createHelper)); + styles.put("footer", poiExcelStylerInterface.createStyleForFooter(workbook, createHelper)); + + XSSFCellStyle footerDateStyle = poiExcelStylerInterface.createStyleForFooter(workbook, createHelper); + footerDateStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_FORMAT)); + styles.put("footer-date", footerDateStyle); + + XSSFCellStyle footerDateTimeStyle = poiExcelStylerInterface.createStyleForFooter(workbook, createHelper); + footerDateTimeStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_TIME_FORMAT)); + styles.put("footer-datetime", footerDateTimeStyle); + } + + + + /******************************************************************************* + ** Starts a new worksheet in the current workbook. Can be called multiple times. + *******************************************************************************/ + @Override + public void start(ExportInput exportInput, List fields, String label, QReportView view) throws QReportingException + { + try + { + ///////////////////////////////////////// + // close previous sheet if one is open // + ///////////////////////////////////////// + closeLastSheetIfOpen(); + + if(currentView != null) + { + this.rowsPerView.put(currentView.getName(), rowNo); + } + + this.currentView = view; + this.exportInput = exportInput; + this.fields = fields; + this.rowNo = 0; + + this.fieldsPerView.put(view.getName(), fields); + + ////////////////////////////////////////// + // start the new sheet as: // + // - a new entry in the zipOutputStream // + // - with a new output stream writer // + // - and with a SpreadsheetWriter // + ////////////////////////////////////////// + zipOutputStream.putNextEntry(new ZipEntry("xl/worksheets/sheet" + this.sheetIndex++ + ".xml")); + activeSheetWriter = new OutputStreamWriter(zipOutputStream); + sheetWriter = new StreamedSheetWriter(activeSheetWriter); + + if(ReportType.PIVOT.equals(view.getType())) + { + writePivotTable(view, ReportUtils.getSourceViewForPivotTableView(views, view)); + } + else + { + sheetWriter.beginSheet(); + + //////////////////////////////////////////////// + // put the title and header rows in the sheet // + //////////////////////////////////////////////// + writeTitleAndHeader(); + } + } + catch(Exception e) + { + throw (new QReportingException("Error starting worksheet", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeTitleAndHeader() throws QReportingException + { + try + { + /////////////// + // title row // + /////////////// + if(StringUtils.hasContent(exportInput.getTitleRow())) + { + sheetWriter.insertRow(rowNo++); + sheetWriter.createCell(0, exportInput.getTitleRow(), styles.get("title").getIndex()); + sheetWriter.endRow(); + } + + //////////////// + // header row // + //////////////// + if(exportInput.getIncludeHeaderRow()) + { + sheetWriter.insertRow(rowNo++); + XSSFCellStyle headerStyle = styles.get("header"); + + int col = 0; + for(QFieldMetaData column : fields) + { + sheetWriter.createCell(col, column.getLabel(), headerStyle.getIndex()); + col++; + } + + sheetWriter.endRow(); + } + } + catch(Exception e) + { + throw (new QReportingException("Error starting Excel report")); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addRecords(List qRecords) throws QReportingException + { + LOG.info("Consuming [" + qRecords.size() + "] records from the pipe"); + + try + { + for(QRecord qRecord : qRecords) + { + writeRecord(qRecord); + } + } + catch(Exception e) + { + LOG.error("Exception generating excel file", e); + try + { + outputStream.close(); + } + catch(IOException ex) + { + LOG.warn("Secondary error closing excel output stream", e); + } + + throw (new QReportingException("Error generating Excel report", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeRecord(QRecord qRecord) throws IOException + { + writeRecord(qRecord, false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeRecord(QRecord qRecord, boolean isFooter) throws IOException + { + sheetWriter.insertRow(rowNo++); + + int styleIndex = -1; + int dateStyleIndex = styles.get("date").getIndex(); + int dateTimeStyleIndex = styles.get("datetime").getIndex(); + if(isFooter) + { + styleIndex = styles.get("footer").getIndex(); + dateStyleIndex = styles.get("footer-date").getIndex(); + dateTimeStyleIndex = styles.get("footer-datetime").getIndex(); + } + + int col = 0; + for(QFieldMetaData field : fields) + { + Serializable value = qRecord.getValue(field.getName()); + + if(value != null) + { + if(value instanceof String s) + { + sheetWriter.createCell(col, s, styleIndex); + } + else if(value instanceof Number n) + { + sheetWriter.createCell(col, n.doubleValue(), styleIndex); + + if(excelCellFormats != null) + { + String format = excelCellFormats.get(field.getName()); + if(format != null) + { + //////////////////////////////////////////////////////////////////////////////////////////// + // todo - so - for this streamed/zip approach, we need to know all styles before we start // + // any sheets. but, right now Report action only calls us with per-sheet styles when // + // it's starting individual sheets. so, we can't quite support this at this time. // + // "just" need to change GenerateReportAction to look up all cell formats for all sheets // + // before preRun is called... and change all existing streamer classes to handle that too // + //////////////////////////////////////////////////////////////////////////////////////////// + // worksheet.style(rowNo, col).format(format).set(); + } + } + } + else if(value instanceof Boolean b) + { + sheetWriter.createCell(col, b, styleIndex); + } + else if(value instanceof Date d) + { + sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex); + } + else if(value instanceof LocalDate d) + { + sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex); + } + else if(value instanceof LocalDateTime d) + { + sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex); + } + else if(value instanceof ZonedDateTime d) + { + sheetWriter.createCell(col, DateUtil.getExcelDate(d.toLocalDateTime()), dateTimeStyleIndex); + } + else if(value instanceof Instant i) + { + sheetWriter.createCell(col, DateUtil.getExcelDate(i.atZone(ZoneId.systemDefault()).toLocalDateTime()), dateTimeStyleIndex); + } + else + { + sheetWriter.createCell(col, ValueUtils.getValueAsString(value), styleIndex); + } + } + + col++; + } + + sheetWriter.endRow(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addTotalsRow(QRecord record) throws QReportingException + { + try + { + writeRecord(record, true); + } + catch(Exception e) + { + throw (new QReportingException("Error adding totals row", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void finish() throws QReportingException + { + try + { + ////////////////////////////////////////////// + // close the last open sheet if one is open // + ////////////////////////////////////////////// + closeLastSheetIfOpen(); + + ///////////////////////////////////////////////////////////////////////////////////// + // so, we DO need a close here, on the zipOutputStream, to finish its "zippiness". // + // even though, doing so also closes the outputStream from the caller that this // + // zipOutputStream is wrapping (and the caller will likely call close on too)... // + ///////////////////////////////////////////////////////////////////////////////////// + zipOutputStream.close(); + } + catch(Exception e) + { + throw (new QReportingException("Error finishing Excel report", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void closeLastSheetIfOpen() throws IOException + { + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we have an active sheet writer: // + // - end the current sheet in the spreadsheet writer (write some closing xml, unless it's a pivot!) // + // - flush the contents through the activeSheetWriter // + // - close the zip entry in the output stream. // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + if(activeSheetWriter != null) + { + if(!ReportType.PIVOT.equals(currentView.getType())) + { + sheetWriter.endSheet(); + } + + activeSheetWriter.flush(); + zipOutputStream.closeEntry(); + } + } + + + + /******************************************************************************* + ** display formats is a map of field name to Excel format strings (e.g., $#,##0.00) + *******************************************************************************/ + @Override + public void setDisplayFormats(Map displayFormats) + { + this.excelCellFormats = new HashMap<>(); + for(Map.Entry entry : displayFormats.entrySet()) + { + String excelFormat = DisplayFormat.getExcelFormat(entry.getValue()); + if(excelFormat != null) + { + excelCellFormats.put(entry.getKey(), excelFormat); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writePivotTable(QReportView pivotTableView, QReportView dataView) throws QReportingException + { + try + { + ////////////////////////////////////////////////////////////////////////////////// + // write the xml file that is the pivot table sheet. // + // note that the ZipEntry here will have been started above in the start method // + ////////////////////////////////////////////////////////////////////////////////// + activeSheetWriter.write(""" + + + + + + + + + + + + + + """); + activeSheetWriter.flush(); + + //////////////////////////////////////////////////////////////////////////// + // start a new zip entry, for this pivot view's cacheDefinition reference // + //////////////////////////////////////////////////////////////////////////// + zipOutputStream.putNextEntry(new ZipEntry(pivotViewToCacheDefinitionReferenceMap.get(pivotTableView.getName()))); + + ///////////////////////////////////////////////////////// + // prepare the xml for each field (e.g., w/ its label) // + ///////////////////////////////////////////////////////// + List cachedFieldElements = new ArrayList<>(); + for(QFieldMetaData column : this.fieldsPerView.get(dataView.getName())) + { + cachedFieldElements.add(String.format(""" + + + + """, column.getLabel())); + } + + ///////////////////////////////////////////////////////////////////////////////////// + // write the xml file that is the pivot cache definition (structure only, no data) // + ///////////////////////////////////////////////////////////////////////////////////// + activeSheetWriter = new OutputStreamWriter(zipOutputStream); + activeSheetWriter.write(String.format(""" + + + + + + + %s + + + """, + StreamedSheetWriter.cleanseValue(labelViewsByName.get(dataView.getName())), + CellReference.convertNumToColString(dataView.getColumns().size() - 1), + rowsPerView.get(dataView.getName()), + dataView.getColumns().size(), + StringUtils.join("\n", cachedFieldElements))); + } + catch(Exception e) + { + throw (new QReportingException("Error writing pivot table", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected PoiExcelStylerInterface getStylerInterface() + { + return (new PlainPoiExcelStyler()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PlainPoiExcelStyler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PlainPoiExcelStyler.java new file mode 100644 index 00000000..15f43ee0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PlainPoiExcelStyler.java @@ -0,0 +1,46 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi; + + +import org.apache.poi.ss.usermodel.CreationHelper; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + + +/******************************************************************************* + ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface. + *******************************************************************************/ +public class PlainPoiExcelStyler implements PoiExcelStylerInterface +{ + + /******************************************************************************* + ** ... sorry, but adding this gives us test coverage on this class, even though + ** we're just deferring to super... + *******************************************************************************/ + @Override + public XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper) + { + return PoiExcelStylerInterface.super.createStyleForHeader(workbook, createHelper); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PoiExcelStylerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PoiExcelStylerInterface.java new file mode 100644 index 00000000..c052e194 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PoiExcelStylerInterface.java @@ -0,0 +1,59 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi; + + +import org.apache.poi.ss.usermodel.CreationHelper; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + + +/******************************************************************************* + ** Interface for classes that know how to apply styles to an Excel stream being + ** built by POI. + *******************************************************************************/ +public interface PoiExcelStylerInterface +{ + /******************************************************************************* + ** + *******************************************************************************/ + default XSSFCellStyle createStyleForTitle(XSSFWorkbook workbook, CreationHelper createHelper) + { + return (workbook.createCellStyle()); + } + + /******************************************************************************* + ** + *******************************************************************************/ + default XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper) + { + return (workbook.createCellStyle()); + } + + /******************************************************************************* + ** + *******************************************************************************/ + default XSSFCellStyle createStyleForFooter(XSSFWorkbook workbook, CreationHelper createHelper) + { + return (workbook.createCellStyle()); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedSheetWriter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedSheetWriter.java new file mode 100644 index 00000000..850a1ecc --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedSheetWriter.java @@ -0,0 +1,242 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi; + + +import java.io.IOException; +import java.io.Writer; +import java.util.HashMap; +import java.util.Map; +import org.apache.poi.ss.util.CellReference; + + +/******************************************************************************* + ** Write excel formatted XML to a Writer. + ** Originally from https://coderanch.com/t/548897/java/Generate-large-excel-POI + *******************************************************************************/ +public class StreamedSheetWriter +{ + private final Writer writer; + private int rowNo; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public StreamedSheetWriter(Writer writer) + { + this.writer = writer; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void beginSheet() throws IOException + { + writer.write(""" + + + """); + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void endSheet() throws IOException + { + writer.write(""" + + """); + } + + + + /******************************************************************************* + ** Insert a new row + ** + ** @param rowNo 0-based row number + *******************************************************************************/ + public void insertRow(int rowNo) throws IOException + { + writer.write("\n"); + this.rowNo = rowNo; + } + + + + /******************************************************************************* + ** Insert row end marker + *******************************************************************************/ + public void endRow() throws IOException + { + writer.write("\n"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void createCell(int columnIndex, String value, int styleIndex) throws IOException + { + String ref = new CellReference(rowNo, columnIndex).formatAsString(); + writer.write(""); + writer.write("" + cleanValue + ""); + writer.write(""); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String cleanseValue(String value) + { + if(value != null) + { + StringBuilder rs = new StringBuilder(); + for(int i = 0; i < value.length(); i++) + { + char c = value.charAt(i); + if(c == '&') + { + rs.append("&"); + } + else if(c == '<') + { + rs.append("<"); + } + else if(c == '>') + { + rs.append(">"); + } + else if(c == '\'') + { + rs.append("'"); + } + else if(c == '"') + { + rs.append("""); + } + else if (c < 32 && c != '\t' && c != '\n') + { + rs.append(' '); + } + else + { + rs.append(c); + } + } + + Map m = new HashMap(); + m.computeIfAbsent("s", (s) -> 3); + + value = rs.toString(); + } + + return (value); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void createCell(int columnIndex, String value) throws IOException + { + createCell(columnIndex, value, -1); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void createCell(int columnIndex, double value, int styleIndex) throws IOException + { + String ref = new CellReference(rowNo, columnIndex).formatAsString(); + writer.write(""); + writer.write("" + value + ""); + writer.write(""); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void createCell(int columnIndex, double value) throws IOException + { + createCell(columnIndex, value, -1); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void createCell(int columnIndex, Boolean value) throws IOException + { + createCell(columnIndex, value, -1); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void createCell(int columnIndex, Boolean value, int styleIndex) throws IOException + { + String ref = new CellReference(rowNo, columnIndex).formatAsString(); + writer.write(""); + if(value != null) + { + writer.write("" + (value ? 1 : 0) + ""); + } + writer.write(""); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java index 841c6d6b..04177cdb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java @@ -47,6 +47,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import org.apache.commons.lang.BooleanUtils; /******************************************************************************* @@ -79,9 +80,11 @@ public class ReplaceAction extends AbstractQActionFunction, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey); + Map, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey, allowNullKeyValuesToEqual); + for(QRecord record : page) { - Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); + Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual); if(keyValues.isPresent()) { if(existingKeys.containsKey(keyValues.get())) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java new file mode 100644 index 00000000..6de52d99 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java @@ -0,0 +1,96 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.tables; + + +import java.io.InputStream; +import java.io.OutputStream; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; + + +/******************************************************************************* + ** Action to do (generally, "mass") storage operations in a backend. + ** + ** e.g., store a (potentially large) file - specifically - by working with it + ** as either an InputStream or OutputStream. + ** + ** May not be implemented in all backends. + ** + *******************************************************************************/ +public class StorageAction +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public OutputStream createOutputStream(StorageInput storageInput) throws QException + { + QBackendModuleInterface qBackendModuleInterface = preAction(storageInput); + QStorageInterface storageInterface = qBackendModuleInterface.getStorageInterface(); + return (storageInterface.createOutputStream(storageInput)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public InputStream getInputStream(StorageInput storageInput) throws QException + { + QBackendModuleInterface qBackendModuleInterface = preAction(storageInput); + QStorageInterface storageInterface = qBackendModuleInterface.getStorageInterface(); + return (storageInterface.getInputStream(storageInput)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QBackendModuleInterface preAction(StorageInput storageInput) throws QException + { + ActionHelper.validateSession(storageInput); + + if(storageInput.getTableName() == null) + { + throw (new QException("Table name was not specified in storage input")); + } + + QTableMetaData table = storageInput.getTable(); + if(table == null) + { + throw (new QException("A table named [" + storageInput.getTableName() + "] was not found in the active QInstance")); + } + + QBackendMetaData backend = storageInput.getBackend(); + QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); + QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(backend); + return (qModule); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java index adab071e..7832344e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java @@ -54,7 +54,7 @@ public class UniqueKeyHelper /******************************************************************************* ** *******************************************************************************/ - public static Map, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List recordList, UniqueKey uniqueKey) throws QException + public static Map, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List recordList, UniqueKey uniqueKey, boolean allowNullKeyValuesToEqual) throws QException { List ukFieldNames = uniqueKey.getFieldNames(); Map, Serializable> existingRecords = new HashMap<>(); @@ -112,7 +112,7 @@ public class UniqueKeyHelper QueryOutput queryOutput = new QueryAction().execute(queryInput); for(QRecord record : queryOutput.getRecords()) { - Optional> keyValues = getKeyValues(table, uniqueKey, record); + Optional> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual); if(keyValues.isPresent()) { existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField())); @@ -128,7 +128,17 @@ public class UniqueKeyHelper /******************************************************************************* ** *******************************************************************************/ - public static Optional> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record) + public static Map, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List recordList, UniqueKey uniqueKey) throws QException + { + return (getExistingKeys(transaction, table, recordList, uniqueKey, false)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Optional> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record, boolean allowNullKeyValuesToEqual) { try { @@ -138,7 +148,19 @@ public class UniqueKeyHelper QFieldMetaData field = table.getField(fieldName); Serializable value = record.getValue(fieldName); Serializable typedValue = ValueUtils.getValueAsFieldType(field.getType(), value); - keyValues.add(typedValue == null ? new NullUniqueKeyValue() : typedValue); + + /////////////////////////////////////////////////////////////////////////////////// + // if null value, look at flag to determine if a null should be used (which will // + // allow keys to match), or a NullUniqueKeyValue, (which will never match) // + /////////////////////////////////////////////////////////////////////////////////// + if(typedValue == null) + { + keyValues.add(allowNullKeyValuesToEqual ? null : new NullUniqueKeyValue()); + } + else + { + keyValues.add(typedValue); + } } return (Optional.of(keyValues)); } @@ -150,6 +172,16 @@ public class UniqueKeyHelper + /******************************************************************************* + ** + *******************************************************************************/ + public static Optional> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record) + { + return (getKeyValues(table, uniqueKey, record, false)); + } + + + /******************************************************************************* ** To make a list of unique key values here behave like they do in an RDBMS ** (which is what we're trying to mimic - which is - 2 null values in a field diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java index f0e6968c..b0e618ce 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java @@ -149,8 +149,7 @@ public class QInstanceHelpContentManager } else if(StringUtils.hasContent(widgetName)) { - processHelpContentForWidget(key, widgetName, slotName, helpContent); - + processHelpContentForWidget(key, widgetName, slotName, roles, helpContent); } } catch(Exception e) @@ -252,7 +251,7 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForWidget(String key, String widgetName, String slotName, QHelpContent helpContent) + private static void processHelpContentForWidget(String key, String widgetName, String slotName, Set roles, QHelpContent helpContent) { QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName); if(!StringUtils.hasContent(slotName)) @@ -265,22 +264,14 @@ public class QInstanceHelpContentManager } else { - Map widgetHelpContent = widget.getHelpContent(); - if(widgetHelpContent == null) - { - widgetHelpContent = new HashMap<>(); - } - if(helpContent != null) { - widgetHelpContent.put(slotName, helpContent); + widget.withHelpContent(slotName, helpContent); } else { - widgetHelpContent.remove(slotName); + widget.removeHelpContent(slotName, roles); } - - widget.setHelpContent(widgetHelpContent); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 65acd19d..85c7e307 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -1383,12 +1383,17 @@ public class QInstanceValidator /////////////////////////////////// // validate steps in the process // /////////////////////////////////// + Set usedStepNames = new HashSet<>(); if(assertCondition(CollectionUtils.nullSafeHasContents(process.getStepList()), "At least 1 step must be defined in process " + processName + ".")) { int index = 0; for(QStepMetaData step : process.getStepList()) { - assertCondition(StringUtils.hasContent(step.getName()), "Missing name for a step at index " + index + " in process " + processName); + if(assertCondition(StringUtils.hasContent(step.getName()), "Missing name for a step at index " + index + " in process " + processName)) + { + assertCondition(!usedStepNames.contains(step.getName()), "Duplicate step name [" + step.getName() + "] in process " + processName); + usedStepNames.add(step.getName()); + } index++; //////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java index 429948d4..3c98715b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java @@ -26,6 +26,7 @@ import java.util.UUID; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; +import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; @@ -139,7 +140,7 @@ public class AbstractActionInput //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // don't return null here (too easy to NPE). instead, if someone wants one of these, create one and give it to them. // //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - asyncJobCallback = new AsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus()); + asyncJobCallback = new NonPersistedAsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus().withJobName(getClass().getSimpleName())); } return asyncJobCallback; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index 18d63f75..bfaad833 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.UUID; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; +import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; @@ -450,7 +451,7 @@ public class RunBackendStepInput extends AbstractActionInput ///////////////////////////////////////////////////////////////////////// // avoid NPE in case we didn't have one of these! create a new one... // ///////////////////////////////////////////////////////////////////////// - asyncJobCallback = new AsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus()); + asyncJobCallback = new NonPersistedAsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus().withJobName(processName + "." + stepName)); } return (asyncJobCallback); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ExportInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ExportInput.java index b774ed73..2ed0ba4e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ExportInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ExportInput.java @@ -22,7 +22,6 @@ package com.kingsrook.qqq.backend.core.model.actions.reporting; -import java.io.OutputStream; import java.util.List; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -37,9 +36,8 @@ public class ExportInput extends AbstractTableActionInput private Integer limit; private List fieldNames; - private String filename; - private ReportFormat reportFormat; - private OutputStream reportOutputStream; + private ReportDestination reportDestination; + private String titleRow; private boolean includeHeaderRow = true; @@ -120,71 +118,6 @@ public class ExportInput extends AbstractTableActionInput - /******************************************************************************* - ** Getter for filename - ** - *******************************************************************************/ - public String getFilename() - { - return filename; - } - - - - /******************************************************************************* - ** Setter for filename - ** - *******************************************************************************/ - public void setFilename(String filename) - { - this.filename = filename; - } - - - - /******************************************************************************* - ** Getter for reportFormat - ** - *******************************************************************************/ - public ReportFormat getReportFormat() - { - return reportFormat; - } - - - - /******************************************************************************* - ** Setter for reportFormat - ** - *******************************************************************************/ - public void setReportFormat(ReportFormat reportFormat) - { - this.reportFormat = reportFormat; - } - - - - /******************************************************************************* - ** Getter for reportOutputStream - ** - *******************************************************************************/ - public OutputStream getReportOutputStream() - { - return reportOutputStream; - } - - - - /******************************************************************************* - ** Setter for reportOutputStream - ** - *******************************************************************************/ - public void setReportOutputStream(OutputStream reportOutputStream) - { - this.reportOutputStream = reportOutputStream; - } - - /******************************************************************************* ** @@ -226,4 +159,36 @@ public class ExportInput extends AbstractTableActionInput this.includeHeaderRow = includeHeaderRow; } + + + /******************************************************************************* + ** Getter for reportDestination + *******************************************************************************/ + public ReportDestination getReportDestination() + { + return (this.reportDestination); + } + + + + /******************************************************************************* + ** Setter for reportDestination + *******************************************************************************/ + public void setReportDestination(ReportDestination reportDestination) + { + this.reportDestination = reportDestination; + } + + + + /******************************************************************************* + ** Fluent setter for reportDestination + *******************************************************************************/ + public ExportInput withReportDestination(ReportDestination reportDestination) + { + this.reportDestination = reportDestination; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportDestination.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportDestination.java new file mode 100644 index 00000000..decc8c5f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportDestination.java @@ -0,0 +1,131 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting; + + +import java.io.OutputStream; + + +/******************************************************************************* + ** Member of report & export Inputs, that wraps details about the destination of + ** where & how the report (or export) is being written. + *******************************************************************************/ +public class ReportDestination +{ + private String filename; + private ReportFormat reportFormat; + private OutputStream reportOutputStream; + + + + /******************************************************************************* + ** Getter for filename + *******************************************************************************/ + public String getFilename() + { + return (this.filename); + } + + + + /******************************************************************************* + ** Setter for filename + *******************************************************************************/ + public void setFilename(String filename) + { + this.filename = filename; + } + + + + /******************************************************************************* + ** Fluent setter for filename + *******************************************************************************/ + public ReportDestination withFilename(String filename) + { + this.filename = filename; + return (this); + } + + + + /******************************************************************************* + ** Getter for reportFormat + *******************************************************************************/ + public ReportFormat getReportFormat() + { + return (this.reportFormat); + } + + + + /******************************************************************************* + ** Setter for reportFormat + *******************************************************************************/ + public void setReportFormat(ReportFormat reportFormat) + { + this.reportFormat = reportFormat; + } + + + + /******************************************************************************* + ** Fluent setter for reportFormat + *******************************************************************************/ + public ReportDestination withReportFormat(ReportFormat reportFormat) + { + this.reportFormat = reportFormat; + return (this); + } + + + + /******************************************************************************* + ** Getter for reportOutputStream + *******************************************************************************/ + public OutputStream getReportOutputStream() + { + return (this.reportOutputStream); + } + + + + /******************************************************************************* + ** Setter for reportOutputStream + *******************************************************************************/ + public void setReportOutputStream(OutputStream reportOutputStream) + { + this.reportOutputStream = reportOutputStream; + } + + + + /******************************************************************************* + ** Fluent setter for reportOutputStream + *******************************************************************************/ + public ReportDestination withReportOutputStream(OutputStream reportOutputStream) + { + this.reportOutputStream = reportOutputStream; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java index c95b1be8..c72056f4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java @@ -25,13 +25,13 @@ package com.kingsrook.qqq.backend.core.model.actions.reporting; import java.util.Locale; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.actions.reporting.CsvExportStreamer; -import com.kingsrook.qqq.backend.core.actions.reporting.ExcelExportStreamer; import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface; import com.kingsrook.qqq.backend.core.actions.reporting.JsonExportStreamer; import com.kingsrook.qqq.backend.core.actions.reporting.ListOfMapsExportStreamer; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.poi.ExcelPoiBasedStreamingExportStreamer; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.utils.StringUtils; -import org.dhatim.fastexcel.Worksheet; +import org.apache.poi.ss.SpreadsheetVersion; /******************************************************************************* @@ -39,15 +39,24 @@ import org.dhatim.fastexcel.Worksheet; *******************************************************************************/ public enum ReportFormat { - XLSX(Worksheet.MAX_ROWS, Worksheet.MAX_COLS, ExcelExportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), - JSON(null, null, JsonExportStreamer::new, "application/json"), - CSV(null, null, CsvExportStreamer::new, "text/csv"), - LIST_OF_MAPS(null, null, ListOfMapsExportStreamer::new, null); + ///////////////////////////////////////////////////////////////////////// + // if we need to fall back to Fastexcel, this was its version of this. // + ///////////////////////////////////////////////////////////////////////// + // XLSX(Worksheet.MAX_ROWS, Worksheet.MAX_COLS, ExcelFastexcelExportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx", true, false, true), + + XLSX(SpreadsheetVersion.EXCEL2007.getMaxRows(), SpreadsheetVersion.EXCEL2007.getMaxColumns(), ExcelPoiBasedStreamingExportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx", true, true, true), + JSON(null, null, JsonExportStreamer::new, "application/json", "json", false, false, true), + CSV(null, null, CsvExportStreamer::new, "text/csv", "csv", false, false, false), + LIST_OF_MAPS(null, null, ListOfMapsExportStreamer::new, null, null, false, false, true); private final Integer maxRows; private final Integer maxCols; private final String mimeType; + private final String extension; + private final boolean isBinary; + private final boolean supportsNativePivotTables; + private final boolean supportsMultipleViews; private final Supplier streamerConstructor; @@ -56,12 +65,16 @@ public enum ReportFormat /******************************************************************************* ** *******************************************************************************/ - ReportFormat(Integer maxRows, Integer maxCols, Supplier streamerConstructor, String mimeType) + ReportFormat(Integer maxRows, Integer maxCols, Supplier streamerConstructor, String mimeType, String extension, boolean isBinary, boolean supportsNativePivotTables, boolean supportsMultipleViews) { this.maxRows = maxRows; this.maxCols = maxCols; this.mimeType = mimeType; this.streamerConstructor = streamerConstructor; + this.extension = extension; + this.isBinary = isBinary; + this.supportsNativePivotTables = supportsNativePivotTables; + this.supportsMultipleViews = supportsMultipleViews; } @@ -128,4 +141,48 @@ public enum ReportFormat { return (streamerConstructor.get()); } + + + + /******************************************************************************* + ** Getter for extension + ** + *******************************************************************************/ + public String getExtension() + { + return extension; + } + + + + /******************************************************************************* + ** Getter for isBinary + ** + *******************************************************************************/ + public boolean getIsBinary() + { + return isBinary; + } + + + + /******************************************************************************* + ** Getter for supportsNativePivotTables + ** + *******************************************************************************/ + public boolean getSupportsNativePivotTables() + { + return supportsNativePivotTables; + } + + + + /******************************************************************************* + ** Getter for supportsMultipleViews + ** + *******************************************************************************/ + public boolean getSupportsMultipleViews() + { + return supportsMultipleViews; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormatPossibleValueEnum.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormatPossibleValueEnum.java new file mode 100644 index 00000000..4dec5da9 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormatPossibleValueEnum.java @@ -0,0 +1,59 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting; + + +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; + + +/******************************************************************************* + ** sub-set of ReportFormats to expose as possible-values in-apps + *******************************************************************************/ +public enum ReportFormatPossibleValueEnum implements PossibleValueEnum +{ + XLSX, + CSV, + JSON; + + public static final String NAME = "reportFormat"; + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getPossibleValueId() + { + return name(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return name(); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java index e712a79e..be901e94 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportInput.java @@ -22,24 +22,28 @@ package com.kingsrook.qqq.backend.core.model.actions.reporting; -import java.io.OutputStream; import java.io.Serializable; import java.util.HashMap; import java.util.Map; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; /******************************************************************************* - ** Input for an Export action + ** Input for a Report action *******************************************************************************/ public class ReportInput extends AbstractTableActionInput { - private String reportName; + private String reportName; + private QReportMetaData reportMetaData; + private Map inputValues; - private String filename; - private ReportFormat reportFormat; - private OutputStream reportOutputStream; + private ReportDestination reportDestination; + + private Supplier overrideExportStreamerSupplier; @@ -111,66 +115,97 @@ public class ReportInput extends AbstractTableActionInput /******************************************************************************* - ** Getter for filename - ** + ** Getter for reportDestination *******************************************************************************/ - public String getFilename() + public ReportDestination getReportDestination() { - return filename; + return (this.reportDestination); } /******************************************************************************* - ** Setter for filename - ** + ** Setter for reportDestination *******************************************************************************/ - public void setFilename(String filename) + public void setReportDestination(ReportDestination reportDestination) { - this.filename = filename; + this.reportDestination = reportDestination; } /******************************************************************************* - ** Getter for reportFormat - ** + ** Fluent setter for reportDestination *******************************************************************************/ - public ReportFormat getReportFormat() + public ReportInput withReportDestination(ReportDestination reportDestination) { - return reportFormat; + this.reportDestination = reportDestination; + return (this); } /******************************************************************************* - ** Setter for reportFormat - ** + ** Getter for reportMetaData *******************************************************************************/ - public void setReportFormat(ReportFormat reportFormat) + public QReportMetaData getReportMetaData() { - this.reportFormat = reportFormat; + return (this.reportMetaData); } /******************************************************************************* - ** Getter for reportOutputStream - ** + ** Setter for reportMetaData *******************************************************************************/ - public OutputStream getReportOutputStream() + public void setReportMetaData(QReportMetaData reportMetaData) { - return reportOutputStream; + this.reportMetaData = reportMetaData; } /******************************************************************************* - ** Setter for reportOutputStream + ** Fluent setter for reportMetaData + *******************************************************************************/ + public ReportInput withReportMetaData(QReportMetaData reportMetaData) + { + this.reportMetaData = reportMetaData; + return (this); + } + + + + /******************************************************************************* + ** Getter for overrideExportStreamerSupplier ** *******************************************************************************/ - public void setReportOutputStream(OutputStream reportOutputStream) + public Supplier getOverrideExportStreamerSupplier() { - this.reportOutputStream = reportOutputStream; + return overrideExportStreamerSupplier; } + + + + /******************************************************************************* + ** Setter for overrideExportStreamerSupplier + ** + *******************************************************************************/ + public void setOverrideExportStreamerSupplier(Supplier overrideExportStreamerSupplier) + { + this.overrideExportStreamerSupplier = overrideExportStreamerSupplier; + } + + + + /******************************************************************************* + ** Fluent setter for overrideExportStreamerSupplier + ** + *******************************************************************************/ + public ReportInput withOverrideExportStreamerSupplier(Supplier overrideExportStreamerSupplier) + { + this.overrideExportStreamerSupplier = overrideExportStreamerSupplier; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java new file mode 100644 index 00000000..7c51ad8e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportOutput.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting; + + +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; + + +/******************************************************************************* + ** Output for a Report action + *******************************************************************************/ +public class ReportOutput extends AbstractActionOutput implements Serializable +{ + private Integer totalRecordCount; + + + + /******************************************************************************* + ** Getter for totalRecordCount + *******************************************************************************/ + public Integer getTotalRecordCount() + { + return (this.totalRecordCount); + } + + + + /******************************************************************************* + ** Setter for totalRecordCount + *******************************************************************************/ + public void setTotalRecordCount(Integer totalRecordCount) + { + this.totalRecordCount = totalRecordCount; + } + + + + /******************************************************************************* + ** Fluent setter for totalRecordCount + *******************************************************************************/ + public ReportOutput withTotalRecordCount(Integer totalRecordCount) + { + this.totalRecordCount = totalRecordCount; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableDefinition.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableDefinition.java new file mode 100644 index 00000000..991597d3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableDefinition.java @@ -0,0 +1,217 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + + +/******************************************************************************* + ** Full definition of a pivot table - its rows, columns, and values. + *******************************************************************************/ +public class PivotTableDefinition implements Cloneable, Serializable +{ + private List rows; + private List columns; + private List values; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected PivotTableDefinition clone() throws CloneNotSupportedException + { + PivotTableDefinition clone = (PivotTableDefinition) super.clone(); + + if(rows != null) + { + clone.rows = new ArrayList<>(); + for(PivotTableGroupBy row : rows) + { + clone.rows.add(row.clone()); + } + } + + if(columns != null) + { + clone.columns = new ArrayList<>(); + for(PivotTableGroupBy column : columns) + { + clone.columns.add(column.clone()); + } + } + + if(values != null) + { + clone.values = new ArrayList<>(); + for(PivotTableValue value : values) + { + clone.values.add(value.clone()); + } + } + + return (clone); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public PivotTableDefinition withRow(PivotTableGroupBy row) + { + if(this.rows == null) + { + this.rows = new ArrayList<>(); + } + this.rows.add(row); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public PivotTableDefinition withColumn(PivotTableGroupBy column) + { + if(this.columns == null) + { + this.columns = new ArrayList<>(); + } + this.columns.add(column); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public PivotTableDefinition withValue(PivotTableValue value) + { + if(this.values == null) + { + this.values = new ArrayList<>(); + } + this.values.add(value); + return (this); + } + + + + /******************************************************************************* + ** Getter for rows + *******************************************************************************/ + public List getRows() + { + return (this.rows); + } + + + + /******************************************************************************* + ** Setter for rows + *******************************************************************************/ + public void setRows(List rows) + { + this.rows = rows; + } + + + + /******************************************************************************* + ** Fluent setter for rows + *******************************************************************************/ + public PivotTableDefinition withRows(List rows) + { + this.rows = rows; + return (this); + } + + + + /******************************************************************************* + ** Getter for columns + *******************************************************************************/ + public List getColumns() + { + return (this.columns); + } + + + + /******************************************************************************* + ** Setter for columns + *******************************************************************************/ + public void setColumns(List columns) + { + this.columns = columns; + } + + + + /******************************************************************************* + ** Fluent setter for columns + *******************************************************************************/ + public PivotTableDefinition withColumns(List columns) + { + this.columns = columns; + return (this); + } + + + + /******************************************************************************* + ** Getter for values + *******************************************************************************/ + public List getValues() + { + return (this.values); + } + + + + /******************************************************************************* + ** Setter for values + *******************************************************************************/ + public void setValues(List values) + { + this.values = values; + } + + + + /******************************************************************************* + ** Fluent setter for values + *******************************************************************************/ + public PivotTableDefinition withValues(List values) + { + this.values = values; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableFunction.java new file mode 100644 index 00000000..a3cd1518 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableFunction.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable; + + +/******************************************************************************* + ** Functions that can be applied to Values in a pivot table. + *******************************************************************************/ +public enum PivotTableFunction +{ + AVERAGE("Average"), + COUNT("Count Values (COUNTA)"), + COUNT_NUMS("Count Numbers (COUNT)"), + MAX("Max"), + MIN("Min"), + PRODUCT("Product"), + STD_DEV("StdDev"), + STD_DEVP("StdDevp"), + SUM("Sum"), + VAR("Var"), + VARP("Varp"); + + + private final String label; + + + + /******************************************************************************* + ** + *******************************************************************************/ + PivotTableFunction(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableGroupBy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableGroupBy.java new file mode 100644 index 00000000..11d1c81e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableGroupBy.java @@ -0,0 +1,143 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable; + + +import java.io.Serializable; + + +/******************************************************************************* + ** Either a row or column grouping in a pivot table. e.g., a field plus + ** sorting details, plus showTotals boolean. + *******************************************************************************/ +public class PivotTableGroupBy implements Cloneable, Serializable +{ + private String fieldName; + private PivotTableOrderBy orderBy; + private boolean showTotals; + + + + /******************************************************************************* + ** Getter for fieldName + *******************************************************************************/ + public String getFieldName() + { + return (this.fieldName); + } + + + + /******************************************************************************* + ** Setter for fieldName + *******************************************************************************/ + public void setFieldName(String fieldName) + { + this.fieldName = fieldName; + } + + + + /******************************************************************************* + ** Fluent setter for fieldName + *******************************************************************************/ + public PivotTableGroupBy withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for orderBy + *******************************************************************************/ + public PivotTableOrderBy getOrderBy() + { + return (this.orderBy); + } + + + + /******************************************************************************* + ** Setter for orderBy + *******************************************************************************/ + public void setOrderBy(PivotTableOrderBy orderBy) + { + this.orderBy = orderBy; + } + + + + /******************************************************************************* + ** Fluent setter for orderBy + *******************************************************************************/ + public PivotTableGroupBy withOrderBy(PivotTableOrderBy orderBy) + { + this.orderBy = orderBy; + return (this); + } + + + + /******************************************************************************* + ** Getter for showTotals + *******************************************************************************/ + public boolean getShowTotals() + { + return (this.showTotals); + } + + + + /******************************************************************************* + ** Setter for showTotals + *******************************************************************************/ + public void setShowTotals(boolean showTotals) + { + this.showTotals = showTotals; + } + + + + /******************************************************************************* + ** Fluent setter for showTotals + *******************************************************************************/ + public PivotTableGroupBy withShowTotals(boolean showTotals) + { + this.showTotals = showTotals; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public PivotTableGroupBy clone() throws CloneNotSupportedException + { + PivotTableGroupBy clone = (PivotTableGroupBy) super.clone(); + return clone; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableOrderBy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableOrderBy.java new file mode 100644 index 00000000..d0dadc5e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableOrderBy.java @@ -0,0 +1,34 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable; + + +import java.io.Serializable; + + +/******************************************************************************* + ** How a group-by (rows or columns) should be sorted. + *******************************************************************************/ +public class PivotTableOrderBy implements Serializable +{ + // todo - implement, but only if POI supports (or we build our own support...) +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableValue.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableValue.java new file mode 100644 index 00000000..035b8855 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/pivottable/PivotTableValue.java @@ -0,0 +1,110 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable; + + +import java.io.Serializable; + + +/******************************************************************************* + ** a value (e.g., field name + function) used in a pivot table + *******************************************************************************/ +public class PivotTableValue implements Cloneable, Serializable +{ + private String fieldName; + private PivotTableFunction function; + + + + /******************************************************************************* + ** Getter for fieldName + *******************************************************************************/ + public String getFieldName() + { + return (this.fieldName); + } + + + + /******************************************************************************* + ** Setter for fieldName + *******************************************************************************/ + public void setFieldName(String fieldName) + { + this.fieldName = fieldName; + } + + + + /******************************************************************************* + ** Fluent setter for fieldName + *******************************************************************************/ + public PivotTableValue withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for function + *******************************************************************************/ + public PivotTableFunction getFunction() + { + return (this.function); + } + + + + /******************************************************************************* + ** Setter for function + *******************************************************************************/ + public void setFunction(PivotTableFunction function) + { + this.function = function; + } + + + + /******************************************************************************* + ** Fluent setter for function + *******************************************************************************/ + public PivotTableValue withFunction(PivotTableFunction function) + { + this.function = function; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public PivotTableValue clone() throws CloneNotSupportedException + { + PivotTableValue clone = (PivotTableValue) super.clone(); + return clone; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java index 192a482d..b6c58ed4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Objects; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -409,9 +410,20 @@ public class QQueryFilter implements Serializable, Cloneable for(Serializable value : criterion.getValues()) { - String valueAsString = ValueUtils.getValueAsString(value); - Serializable interpretedValue = variableInterpreter.interpretForObject(valueAsString); - newValues.add(interpretedValue); + if(value instanceof AbstractFilterExpression) + { + ///////////////////////////////////////////////////////////////////////// + // todo - do we want to try to interpret values within the expression? // + // e.g., greater than now minus ${input.noOfDays} // + ///////////////////////////////////////////////////////////////////////// + newValues.add(value); + } + else + { + String valueAsString = ValueUtils.getValueAsString(value); + Serializable interpretedValue = variableInterpreter.interpretForObject(valueAsString); + newValues.add(interpretedValue); + } } criterion.setValues(newValues); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java index 81944a6a..354a3bde 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java @@ -39,7 +39,8 @@ public class ReplaceInput extends AbstractTableActionInput private UniqueKey key; private List records; private QQueryFilter filter; - private boolean performDeletes = true; + private boolean performDeletes = true; + private boolean allowNullKeyValuesToEqual = false; private boolean omitDmlAudit = false; @@ -239,4 +240,35 @@ public class ReplaceInput extends AbstractTableActionInput return (this); } + + + /******************************************************************************* + ** Getter for allowNullKeyValuesToEqual + *******************************************************************************/ + public boolean getAllowNullKeyValuesToEqual() + { + return (this.allowNullKeyValuesToEqual); + } + + + + /******************************************************************************* + ** Setter for allowNullKeyValuesToEqual + *******************************************************************************/ + public void setAllowNullKeyValuesToEqual(boolean allowNullKeyValuesToEqual) + { + this.allowNullKeyValuesToEqual = allowNullKeyValuesToEqual; + } + + + + /******************************************************************************* + ** Fluent setter for allowNullKeyValuesToEqual + *******************************************************************************/ + public ReplaceInput withAllowNullKeyValuesToEqual(boolean allowNullKeyValuesToEqual) + { + this.allowNullKeyValuesToEqual = allowNullKeyValuesToEqual; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java new file mode 100644 index 00000000..5407faeb --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.tables.storage; + + +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; + + +/******************************************************************************* + ** Input for Storage actions. + *******************************************************************************/ +public class StorageInput extends AbstractTableActionInput +{ + private String reference; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public StorageInput(String storageTableName) + { + super(); + setTableName(storageTableName); + } + + + + /******************************************************************************* + ** Getter for reference + *******************************************************************************/ + public String getReference() + { + return (this.reference); + } + + + + /******************************************************************************* + ** Setter for reference + *******************************************************************************/ + public void setReference(String reference) + { + this.reference = reference; + } + + + + /******************************************************************************* + ** Fluent setter for reference + *******************************************************************************/ + public StorageInput withReference(String reference) + { + this.reference = reference; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/AlertData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/AlertData.java new file mode 100644 index 00000000..28b60929 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/AlertData.java @@ -0,0 +1,139 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.dashboard.widgets; + + +/******************************************************************************* + ** Model containing datastructure expected by frontend alert widget + ** + *******************************************************************************/ +public class AlertData extends QWidgetData +{ + public enum AlertType + { + ERROR, + SUCCESS, + WARNING + } + + + + private String html; + private AlertType alertType; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public AlertData() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public AlertData(AlertType alertType, String html) + { + setHtml(html); + setAlertType(alertType); + } + + + + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public String getType() + { + return WidgetType.ALERT.getType(); + } + + + + /******************************************************************************* + ** Getter for html + ** + *******************************************************************************/ + public String getHtml() + { + return html; + } + + + + /******************************************************************************* + ** Setter for html + ** + *******************************************************************************/ + public void setHtml(String html) + { + this.html = html; + } + + + + /******************************************************************************* + ** Fluent setter for html + ** + *******************************************************************************/ + public AlertData withHtml(String html) + { + this.html = html; + return (this); + } + + + + /******************************************************************************* + ** Getter for alertType + *******************************************************************************/ + public AlertType getAlertType() + { + return (this.alertType); + } + + + + /******************************************************************************* + ** Setter for alertType + *******************************************************************************/ + public void setAlertType(AlertType alertType) + { + this.alertType = alertType; + } + + + + /******************************************************************************* + ** Fluent setter for alertType + *******************************************************************************/ + public AlertData withAlertType(AlertType alertType) + { + this.alertType = alertType; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java index 421164a5..8b4061de 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java @@ -27,6 +27,7 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets; *******************************************************************************/ public enum WidgetType { + ALERT("alert"), BAR_CHART("barChart"), CHART("chart"), CHILD_RECORD_LIST("childRecordList"), @@ -50,7 +51,9 @@ public enum WidgetType USA_MAP("usaMap"), COMPOSITE("composite"), DATA_BAG_VIEWER("dataBagViewer"), - SCRIPT_VIEWER("scriptViewer"); + SCRIPT_VIEWER("scriptViewer"), + REPORT_SETUP("reportSetup"), + PIVOT_TABLE_SETUP("pivotTableSetup"); private final String type; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java index 8744362c..c13b87f5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java @@ -26,6 +26,7 @@ import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; @@ -88,6 +89,11 @@ public @interface QField *******************************************************************************/ ValueTooLongBehavior valueTooLongBehavior() default ValueTooLongBehavior.PASS_THROUGH; + /******************************************************************************* + ** + *******************************************************************************/ + DynamicDefaultValueBehavior dynamicDefaultValueBehavior() default DynamicDefaultValueBehavior.NONE; + ////////////////////////////////////////////////////////////////////////////////////////// // new attributes here likely need implementation in QFieldMetaData.constructFromGetter // ////////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java index e09673a3..06a35280 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java @@ -27,11 +27,15 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import com.google.common.reflect.ClassPath; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -43,6 +47,29 @@ public class MetaDataProducerHelper { private static final QLogger LOG = QLogger.getLogger(MetaDataProducerHelper.class); + private static Map, Integer> comparatorValuesByType = new HashMap<>(); + private static Integer defaultComparatorValue; + + static + { + //////////////////////////////////////////////////////////////////////////////////////// + // define how we break ties in sort-order based on the meta-dta type. e.g., do apps // + // after all other types (as apps often try to get other types from the instance) // + // also - do backends earlier than others (e.g., tables may expect backends to exist) // + // any types not in the map get the default value. // + //////////////////////////////////////////////////////////////////////////////////////// + comparatorValuesByType.put(QBackendMetaData.class, 1); + + ///////////////////////////////////// + // unspecified ones will come here // + ///////////////////////////////////// + defaultComparatorValue = 10; + + comparatorValuesByType.put(QJoinMetaData.class, 21); + comparatorValuesByType.put(QWidgetMetaData.class, 22); + comparatorValuesByType.put(QAppMetaData.class, 23); + } + /******************************************************************************* @@ -102,11 +129,9 @@ public class MetaDataProducerHelper } } - //////////////////////////////////////////////////////////////////////////////////////////// - // sort them by sort order, then by the type that they return - specifically - doing apps // - // after all other types (as apps often try to get other types from the instance) // - // also - do backends earlier than others (e.g., tables may expect backends to exist) // - //////////////////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////////////////////// + // sort them by sort order, then by the type that they return, as set up in the static map // + ///////////////////////////////////////////////////////////////////////////////////////////// producers.sort(Comparator .comparing((MetaDataProducerInterface p) -> p.getSortOrder()) .thenComparing((MetaDataProducerInterface p) -> @@ -114,18 +139,7 @@ public class MetaDataProducerHelper try { Class outputType = p.getClass().getMethod("produce", QInstance.class).getReturnType(); - if(outputType.equals(QAppMetaData.class)) - { - return (2); - } - else if(outputType.equals(QBackendMetaData.class)) - { - return (0); - } - else - { - return (1); - } + return comparatorValuesByType.getOrDefault(outputType, defaultComparatorValue); } catch(Exception e) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java index eb63b7f4..628fa870 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java @@ -155,18 +155,6 @@ public class QBackendMetaData implements TopLevelMetaDataInterface - /******************************************************************************* - ** Fluent setter, returning generically, to help sub-class fluent flows - *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withBackendType(String backendType) - { - this.backendType = backendType; - return (T) this; - } - - - /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java index 03688222..e7793b03 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java @@ -24,11 +24,15 @@ package com.kingsrook.qqq.backend.core.model.metadata.dashboard; import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.core.instances.QInstanceHelpContentManager; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole; import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; @@ -61,7 +65,7 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface protected Map icons; - protected Map helpContent; + protected Map> helpContent; protected Map defaultValues = new LinkedHashMap<>(); @@ -691,10 +695,11 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface } + /******************************************************************************* ** Getter for helpContent *******************************************************************************/ - public Map getHelpContent() + public Map> getHelpContent() { return (this.helpContent); } @@ -704,7 +709,7 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface /******************************************************************************* ** Setter for helpContent *******************************************************************************/ - public void setHelpContent(Map helpContent) + public void setHelpContent(Map> helpContent) { this.helpContent = helpContent; } @@ -714,11 +719,49 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface /******************************************************************************* ** Fluent setter for helpContent *******************************************************************************/ - public QWidgetMetaData withHelpContent(Map helpContent) + public QWidgetMetaData withHelpContent(Map> helpContent) { this.helpContent = helpContent; return (this); } + + /******************************************************************************* + ** Fluent setter for adding 1 helpContent (for a slot) + *******************************************************************************/ + public QWidgetMetaData withHelpContent(String slot, QHelpContent helpContent) + { + if(this.helpContent == null) + { + this.helpContent = new HashMap<>(); + } + + List listForSlot = this.helpContent.computeIfAbsent(slot, (k) -> new ArrayList<>()); + QInstanceHelpContentManager.putHelpContentInList(helpContent, listForSlot); + + return (this); + } + + + + /******************************************************************************* + ** remove a helpContent for a slot based on its set of roles + *******************************************************************************/ + public void removeHelpContent(String slot, Set roles) + { + if(this.helpContent == null) + { + return; + } + + List listForSlot = this.helpContent.get(slot); + if(listForSlot == null) + { + return; + } + + QInstanceHelpContentManager.removeHelpContentByRoleSetFromList(roles, listForSlot); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java index 1c702e39..ed8af4e3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java @@ -25,10 +25,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.dashboard; import java.io.Serializable; import java.util.List; import java.util.Map; +import java.util.Set; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole; import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; @@ -235,7 +237,7 @@ public interface QWidgetMetaDataInterface extends MetaDataWithPermissionRules, T /******************************************************************************* ** *******************************************************************************/ - default Map getHelpContent() + default Map> getHelpContent() { return (null); } @@ -244,11 +246,29 @@ public interface QWidgetMetaDataInterface extends MetaDataWithPermissionRules, T /******************************************************************************* ** *******************************************************************************/ - default void setHelpContent(Map helpContent) + default void setHelpContent(Map> helpContent) { LOG.debug("Setting help content in a widgetMetaData type that doesn't support it (because it didn't override the getter/setter)"); } + /******************************************************************************* + ** + *******************************************************************************/ + default QWidgetMetaDataInterface withHelpContent(String slot, QHelpContent helpContent) + { + LOG.debug("Setting help content in a widgetMetaData type that doesn't support it (because it didn't override the getter/setter)"); + return (this); + } + + /******************************************************************************* + ** remove a helpContent for a slot based on its set of roles + *******************************************************************************/ + default void removeHelpContent(String slot, Set roles) + { + LOG.debug("Setting help content in a widgetMetaData type that doesn't support it (because it didn't override the getter/setter)"); + } + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownData.java index d5446dea..37bd9d23 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownData.java @@ -28,6 +28,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.dashboard; *******************************************************************************/ public class WidgetDropdownData { + private String name; private String possibleValueSourceName; private String foreignKeyFieldName; private String label; @@ -44,6 +45,9 @@ public class WidgetDropdownData //////////////////////////////////////////////////////////////////////////////////////////////// private String labelForNullValue; + private WidgetDropdownType type = WidgetDropdownType.POSSIBLE_VALUE_SOURCE; + + /******************************************************************************* ** Getter for possibleValueSourceName @@ -366,4 +370,65 @@ public class WidgetDropdownData } + + /******************************************************************************* + ** Getter for type + *******************************************************************************/ + public WidgetDropdownType getType() + { + return (this.type); + } + + + + /******************************************************************************* + ** Setter for type + *******************************************************************************/ + public void setType(WidgetDropdownType type) + { + this.type = type; + } + + + + /******************************************************************************* + ** Fluent setter for type + *******************************************************************************/ + public WidgetDropdownData withType(WidgetDropdownType type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public WidgetDropdownData withName(String name) + { + this.name = name; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/PlainExcelStyler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownType.java similarity index 82% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/PlainExcelStyler.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownType.java index 3fdd189a..be47959f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/PlainExcelStyler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownType.java @@ -19,13 +19,15 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting; +package com.kingsrook.qqq.backend.core.model.metadata.dashboard; /******************************************************************************* - ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface. + ** Possible types for widget dropdowns + ** *******************************************************************************/ -public class PlainExcelStyler implements ExcelStylerInterface +public enum WidgetDropdownType { - + POSSIBLE_VALUE_SOURCE, + DATE_PICKER } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehavior.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehavior.java index 5345a53d..64104583 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehavior.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehavior.java @@ -93,7 +93,13 @@ public class DateTimeDisplayValueBehavior implements FieldDisplayBehavior applyCreateDate(action, recordList, table, field); case MODIFY_DATE -> applyModifyDate(action, recordList, table, field); + case USER_ID -> applyUserId(action, recordList, table, field); default -> throw new IllegalStateException("Unexpected enum value: " + this); } } @@ -131,6 +136,27 @@ public enum DynamicDefaultValueBehavior implements FieldBehavior recordList, QTableMetaData table, QFieldMetaData field) + { + String fieldName = field.getName(); + String userId = ObjectUtils.tryElse(() -> QContext.getQSession().getUser().getIdReference(), null); + if(StringUtils.hasContent(userId)) + { + for(QRecord record : CollectionUtils.nonNullList(recordList)) + { + if(!StringUtils.hasContent(record.getValueString(fieldName))) + { + record.setValue(field.getName(), userId); + } + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java index 8a1447be..d78d469d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java @@ -221,11 +221,16 @@ public class QFieldMetaData implements Cloneable setMaxLength(fieldAnnotation.maxLength()); } - if(fieldAnnotation.valueTooLongBehavior() != ValueTooLongBehavior.PASS_THROUGH) + if(fieldAnnotation.valueTooLongBehavior() != ValueTooLongBehavior.values()[0].getDefault()) { withBehavior(fieldAnnotation.valueTooLongBehavior()); } + if(fieldAnnotation.dynamicDefaultValueBehavior() != DynamicDefaultValueBehavior.values()[0].getDefault()) + { + withBehavior(fieldAnnotation.dynamicDefaultValueBehavior()); + } + if(StringUtils.hasContent(fieldAnnotation.defaultValue())) { ValueUtils.getValueAsFieldType(this.type, fieldAnnotation.defaultValue()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java index 4d5e5725..4312949f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java @@ -59,9 +59,9 @@ public class QFrontendWidgetMetaData private boolean showReloadButton = false; private boolean showExportButton = false; - protected Map icons; - protected Map helpContent; - protected Map defaultValues; + protected Map icons; + protected Map> helpContent; + protected Map defaultValues; private final boolean hasPermission; @@ -273,7 +273,7 @@ public class QFrontendWidgetMetaData ** Getter for helpContent ** *******************************************************************************/ - public Map getHelpContent() + public Map> getHelpContent() { return helpContent; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportView.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportView.java index 1247ddfe..fa929c0d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportView.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportView.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.reporting; import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; @@ -40,11 +41,11 @@ public class QReportView implements Cloneable private ReportType type; private String titleFormat; private List titleFields; - private List pivotFields; + private List summaryFields; - private boolean includeHeaderRow = true; - private boolean includeTotalRow = false; - private boolean includePivotSubTotals = false; + private boolean includeHeaderRow = true; + private boolean includeTotalRow = false; + private boolean includeSummarySubTotals = false; private List columns; private List orderByFields; @@ -52,6 +53,9 @@ public class QReportView implements Cloneable private QCodeReference recordTransformStep; private QCodeReference viewCustomizer; + private String pivotTableSourceViewName; + private PivotTableDefinition pivotTableDefinition; + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Note: This class is Cloneable - think about if new fields added here need deep-copied in the clone method! // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -297,34 +301,34 @@ public class QReportView implements Cloneable /******************************************************************************* - ** Getter for pivotFields + ** Getter for summaryFields ** *******************************************************************************/ - public List getPivotFields() + public List getSummaryFields() { - return pivotFields; + return summaryFields; } /******************************************************************************* - ** Setter for pivotFields + ** Setter for summaryFields ** *******************************************************************************/ - public void setPivotFields(List pivotFields) + public void setSummaryFields(List summaryFields) { - this.pivotFields = pivotFields; + this.summaryFields = summaryFields; } /******************************************************************************* - ** Fluent setter for pivotFields + ** Fluent setter for summaryFields ** *******************************************************************************/ - public QReportView withPivotFields(List pivotFields) + public QReportView withSummaryFields(List summaryFields) { - this.pivotFields = pivotFields; + this.summaryFields = summaryFields; return (this); } @@ -399,34 +403,34 @@ public class QReportView implements Cloneable /******************************************************************************* - ** Getter for pivotSubTotals + ** Getter for summarySubTotals ** *******************************************************************************/ - public boolean getIncludePivotSubTotals() + public boolean getIncludeSummarySubTotals() { - return includePivotSubTotals; + return includeSummarySubTotals; } /******************************************************************************* - ** Setter for pivotSubTotals + ** Setter for summarySubTotals ** *******************************************************************************/ - public void setIncludePivotSubTotals(boolean includePivotSubTotals) + public void setIncludeSummarySubTotals(boolean includeSummarySubTotals) { - this.includePivotSubTotals = includePivotSubTotals; + this.includeSummarySubTotals = includeSummarySubTotals; } /******************************************************************************* - ** Fluent setter for pivotSubTotals + ** Fluent setter for summarySubTotals ** *******************************************************************************/ - public QReportView withIncludePivotSubTotals(boolean pivotSubTotals) + public QReportView withIncludeSummarySubTotals(boolean summarySubTotals) { - this.includePivotSubTotals = pivotSubTotals; + this.includeSummarySubTotals = summarySubTotals; return (this); } @@ -602,9 +606,9 @@ public class QReportView implements Cloneable clone.setTitleFields(new ArrayList<>(titleFields)); } - if(pivotFields != null) + if(summaryFields != null) { - clone.setPivotFields(new ArrayList<>(pivotFields)); + clone.setSummaryFields(new ArrayList<>(summaryFields)); } if(columns != null) @@ -624,4 +628,67 @@ public class QReportView implements Cloneable throw new AssertionError(); } } + + + + /******************************************************************************* + ** Getter for pivotTableSourceViewName + *******************************************************************************/ + public String getPivotTableSourceViewName() + { + return (this.pivotTableSourceViewName); + } + + + + /******************************************************************************* + ** Setter for pivotTableSourceViewName + *******************************************************************************/ + public void setPivotTableSourceViewName(String pivotTableSourceViewName) + { + this.pivotTableSourceViewName = pivotTableSourceViewName; + } + + + + /******************************************************************************* + ** Fluent setter for pivotTableSourceViewName + *******************************************************************************/ + public QReportView withPivotTableSourceViewName(String pivotTableSourceViewName) + { + this.pivotTableSourceViewName = pivotTableSourceViewName; + return (this); + } + + + + /******************************************************************************* + ** Getter for pivotTableDefinition + *******************************************************************************/ + public PivotTableDefinition getPivotTableDefinition() + { + return (this.pivotTableDefinition); + } + + + + /******************************************************************************* + ** Setter for pivotTableDefinition + *******************************************************************************/ + public void setPivotTableDefinition(PivotTableDefinition pivotTableDefinition) + { + this.pivotTableDefinition = pivotTableDefinition; + } + + + + /******************************************************************************* + ** Fluent setter for pivotTableDefinition + *******************************************************************************/ + public QReportView withPivotTableDefinition(PivotTableDefinition pivotTableDefinition) + { + this.pivotTableDefinition = pivotTableDefinition; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/ReportType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/ReportType.java index 8492537a..22b5cb67 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/ReportType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/ReportType.java @@ -28,6 +28,6 @@ package com.kingsrook.qqq.backend.core.model.metadata.reporting; public enum ReportType { TABLE, // e.g., raw data in tabular form. - SUMMARY, // e.g., summaries computed within QQQ - PIVOT // e.g., a true spreadsheet pivot. Not initially supported... + SUMMARY, // e.g., summaries computed within QQQ. + PIVOT // e.g., a true spreadsheet pivot. } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/RenderedReport.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/RenderedReport.java new file mode 100644 index 00000000..19ef870d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/RenderedReport.java @@ -0,0 +1,506 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; + + +/******************************************************************************* + ** Entity bean for the rendered report table + *******************************************************************************/ +public class RenderedReport extends QRecordEntity +{ + public static final String TABLE_NAME = "renderedReport"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, dynamicDefaultValueBehavior = DynamicDefaultValueBehavior.USER_ID) + private String userId; + + @QField(possibleValueSourceName = SavedReport.TABLE_NAME) + private Integer savedReportId; + + @QField(possibleValueSourceName = RenderedReportStatus.NAME, label = "Status") + private Integer renderedReportStatusId; + + @QField(maxLength = 40, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String jobUuid; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String resultPath; + + @QField(maxLength = 10, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = ReportFormatPossibleValueEnum.NAME) + private String reportFormat; + + @QField() + private Instant startTime; + + @QField() + private Instant endTime; + + @QField(displayFormat = DisplayFormat.COMMAS) + private Integer rowCount; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS) + private String errorMessage; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public RenderedReport() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public RenderedReport(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public RenderedReport withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public RenderedReport withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public RenderedReport withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for userId + *******************************************************************************/ + public String getUserId() + { + return (this.userId); + } + + + + /******************************************************************************* + ** Setter for userId + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + *******************************************************************************/ + public RenderedReport withUserId(String userId) + { + this.userId = userId; + return (this); + } + + + + /******************************************************************************* + ** Getter for savedReportId + *******************************************************************************/ + public Integer getSavedReportId() + { + return (this.savedReportId); + } + + + + /******************************************************************************* + ** Setter for savedReportId + *******************************************************************************/ + public void setSavedReportId(Integer savedReportId) + { + this.savedReportId = savedReportId; + } + + + + /******************************************************************************* + ** Fluent setter for savedReportId + *******************************************************************************/ + public RenderedReport withSavedReportId(Integer savedReportId) + { + this.savedReportId = savedReportId; + return (this); + } + + + + /******************************************************************************* + ** Getter for renderedReportStatusId + *******************************************************************************/ + public Integer getRenderedReportStatusId() + { + return (this.renderedReportStatusId); + } + + + + /******************************************************************************* + ** Setter for renderedReportStatusId + *******************************************************************************/ + public void setRenderedReportStatusId(Integer renderedReportStatusId) + { + this.renderedReportStatusId = renderedReportStatusId; + } + + + + /******************************************************************************* + ** Fluent setter for renderedReportStatusId + *******************************************************************************/ + public RenderedReport withRenderedReportStatusId(Integer renderedReportStatusId) + { + this.renderedReportStatusId = renderedReportStatusId; + return (this); + } + + + + /******************************************************************************* + ** Getter for jobUuid + *******************************************************************************/ + public String getJobUuid() + { + return (this.jobUuid); + } + + + + /******************************************************************************* + ** Setter for jobUuid + *******************************************************************************/ + public void setJobUuid(String jobUuid) + { + this.jobUuid = jobUuid; + } + + + + /******************************************************************************* + ** Fluent setter for jobUuid + *******************************************************************************/ + public RenderedReport withJobUuid(String jobUuid) + { + this.jobUuid = jobUuid; + return (this); + } + + + + /******************************************************************************* + ** Getter for resultPath + *******************************************************************************/ + public String getResultPath() + { + return (this.resultPath); + } + + + + /******************************************************************************* + ** Setter for resultPath + *******************************************************************************/ + public void setResultPath(String resultPath) + { + this.resultPath = resultPath; + } + + + + /******************************************************************************* + ** Fluent setter for resultPath + *******************************************************************************/ + public RenderedReport withResultPath(String resultPath) + { + this.resultPath = resultPath; + return (this); + } + + + + /******************************************************************************* + ** Getter for reportFormat + *******************************************************************************/ + public String getReportFormat() + { + return (this.reportFormat); + } + + + + /******************************************************************************* + ** Setter for reportFormat + *******************************************************************************/ + public void setReportFormat(String reportFormat) + { + this.reportFormat = reportFormat; + } + + + + /******************************************************************************* + ** Fluent setter for reportFormat + *******************************************************************************/ + public RenderedReport withReportFormat(String reportFormat) + { + this.reportFormat = reportFormat; + return (this); + } + + + + /******************************************************************************* + ** Getter for startTime + *******************************************************************************/ + public Instant getStartTime() + { + return (this.startTime); + } + + + + /******************************************************************************* + ** Setter for startTime + *******************************************************************************/ + public void setStartTime(Instant startTime) + { + this.startTime = startTime; + } + + + + /******************************************************************************* + ** Fluent setter for startTime + *******************************************************************************/ + public RenderedReport withStartTime(Instant startTime) + { + this.startTime = startTime; + return (this); + } + + + + /******************************************************************************* + ** Getter for endTime + *******************************************************************************/ + public Instant getEndTime() + { + return (this.endTime); + } + + + + /******************************************************************************* + ** Setter for endTime + *******************************************************************************/ + public void setEndTime(Instant endTime) + { + this.endTime = endTime; + } + + + + /******************************************************************************* + ** Fluent setter for endTime + *******************************************************************************/ + public RenderedReport withEndTime(Instant endTime) + { + this.endTime = endTime; + return (this); + } + + + + /******************************************************************************* + ** Getter for rowCount + *******************************************************************************/ + public Integer getRowCount() + { + return (this.rowCount); + } + + + + /******************************************************************************* + ** Setter for rowCount + *******************************************************************************/ + public void setRowCount(Integer rowCount) + { + this.rowCount = rowCount; + } + + + + /******************************************************************************* + ** Fluent setter for rowCount + *******************************************************************************/ + public RenderedReport withRowCount(Integer rowCount) + { + this.rowCount = rowCount; + return (this); + } + + + + /******************************************************************************* + ** Getter for errorMessage + *******************************************************************************/ + public String getErrorMessage() + { + return (this.errorMessage); + } + + + + /******************************************************************************* + ** Setter for errorMessage + *******************************************************************************/ + public void setErrorMessage(String errorMessage) + { + this.errorMessage = errorMessage; + } + + + + /******************************************************************************* + ** Fluent setter for errorMessage + *******************************************************************************/ + public RenderedReport withErrorMessage(String errorMessage) + { + this.errorMessage = errorMessage; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/RenderedReportStatus.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/RenderedReportStatus.java new file mode 100644 index 00000000..b407fc07 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/RenderedReportStatus.java @@ -0,0 +1,96 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum RenderedReportStatus implements PossibleValueEnum +{ + RUNNING(1, "Running"), + COMPLETE(2, "Complete"), + FAILED(3, "Failed"); + + public static final String NAME = "renderedReportStatus"; + + private final Integer id; + private final String label; + + + + /******************************************************************************* + ** + *******************************************************************************/ + RenderedReportStatus(int id, String label) + { + this.id = id; + this.label = label; + } + + + + /******************************************************************************* + ** Getter for id + ** + *******************************************************************************/ + public Integer getId() + { + return id; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Integer getPossibleValueId() + { + return id; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return label; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportColumn.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportColumn.java new file mode 100644 index 00000000..44518fe2 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportColumn.java @@ -0,0 +1,98 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.io.Serializable; + + +/******************************************************************************* + ** single entry in ReportColumns object - as part of SavedReport + *******************************************************************************/ +public class ReportColumn implements Serializable +{ + private String name; + private Boolean isVisible; + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public ReportColumn withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for isVisible + *******************************************************************************/ + public Boolean getIsVisible() + { + return (this.isVisible); + } + + + + /******************************************************************************* + ** Setter for isVisible + *******************************************************************************/ + public void setIsVisible(Boolean isVisible) + { + this.isVisible = isVisible; + } + + + + /******************************************************************************* + ** Fluent setter for isVisible + *******************************************************************************/ + public ReportColumn withIsVisible(Boolean isVisible) + { + this.isVisible = isVisible; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportColumns.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportColumns.java new file mode 100644 index 00000000..a18e1dd9 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportColumns.java @@ -0,0 +1,117 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** type of object expected to be in the SavedReport columnsJSON field + *******************************************************************************/ +public class ReportColumns implements Serializable +{ + private List columns; + + + + /******************************************************************************* + ** Getter for columns + *******************************************************************************/ + public List getColumns() + { + return (this.columns); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public List extractVisibleColumns() + { + return CollectionUtils.nonNullList(getColumns()).stream() + ////////////////////////////////////////////////////// + // if isVisible is missing, we assume it to be true // + ////////////////////////////////////////////////////// + .filter(rc -> rc.getIsVisible() == null || rc.getIsVisible()) + .filter(rc -> StringUtils.hasContent(rc.getName())) + .filter(rc -> !rc.getName().startsWith("__check")) + .toList(); + } + + + + /******************************************************************************* + ** Setter for columns + *******************************************************************************/ + public void setColumns(List columns) + { + this.columns = columns; + } + + + + /******************************************************************************* + ** Fluent setter for columns + *******************************************************************************/ + public ReportColumns withColumns(List columns) + { + this.columns = columns; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter to add 1 column + *******************************************************************************/ + public ReportColumns withColumn(ReportColumn column) + { + if(this.columns == null) + { + this.columns = new ArrayList<>(); + } + this.columns.add(column); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter to add 1 column w/ just a name + *******************************************************************************/ + public ReportColumns withColumn(String name) + { + if(this.columns == null) + { + this.columns = new ArrayList<>(); + } + this.columns.add(new ReportColumn().withName(name)); + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReport.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReport.java new file mode 100644 index 00000000..190efbd1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReport.java @@ -0,0 +1,386 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; + + +/******************************************************************************* + ** Entity bean for the saved report table + *******************************************************************************/ +public class SavedReport extends QRecordEntity +{ + public static final String TABLE_NAME = "savedReport"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(isRequired = true, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS, label = "Report Name") + private String label; + + @QField(possibleValueSourceName = TablesPossibleValueSourceMetaDataProvider.NAME, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, label = "Table", isRequired = true) + private String tableName; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, dynamicDefaultValueBehavior = DynamicDefaultValueBehavior.USER_ID) + private String userId; + + @QField(label = "Query Filter") + private String queryFilterJson; + + @QField(label = "Columns") + private String columnsJson; + + @QField(label = "Input Fields") + private String inputFieldsJson; + + @QField(label = "Pivot Table") + private String pivotTableJson; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SavedReport() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SavedReport(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + /******************************************************************************* + ** Getter for id + ** + *******************************************************************************/ + public Integer getId() + { + return id; + } + + + + /******************************************************************************* + ** Setter for id + ** + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Getter for createDate + ** + *******************************************************************************/ + public Instant getCreateDate() + { + return createDate; + } + + + + /******************************************************************************* + ** Setter for createDate + ** + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Getter for modifyDate + ** + *******************************************************************************/ + public Instant getModifyDate() + { + return modifyDate; + } + + + + /******************************************************************************* + ** Setter for modifyDate + ** + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** Setter for label + ** + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + ** + *******************************************************************************/ + public SavedReport withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for tableName + ** + *******************************************************************************/ + public String getTableName() + { + return tableName; + } + + + + /******************************************************************************* + ** Setter for tableName + ** + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + ** + *******************************************************************************/ + public SavedReport withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for userId + ** + *******************************************************************************/ + public String getUserId() + { + return userId; + } + + + + /******************************************************************************* + ** Setter for userId + ** + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + ** + *******************************************************************************/ + public SavedReport withUserId(String userId) + { + this.userId = userId; + return (this); + } + + + + /******************************************************************************* + ** Getter for queryFilterJson + *******************************************************************************/ + public String getQueryFilterJson() + { + return (this.queryFilterJson); + } + + + + /******************************************************************************* + ** Setter for queryFilterJson + *******************************************************************************/ + public void setQueryFilterJson(String queryFilterJson) + { + this.queryFilterJson = queryFilterJson; + } + + + + /******************************************************************************* + ** Fluent setter for queryFilterJson + *******************************************************************************/ + public SavedReport withQueryFilterJson(String queryFilterJson) + { + this.queryFilterJson = queryFilterJson; + return (this); + } + + + + /******************************************************************************* + ** Getter for columnsJson + *******************************************************************************/ + public String getColumnsJson() + { + return (this.columnsJson); + } + + + + /******************************************************************************* + ** Setter for columnsJson + *******************************************************************************/ + public void setColumnsJson(String columnsJson) + { + this.columnsJson = columnsJson; + } + + + + /******************************************************************************* + ** Fluent setter for columnsJson + *******************************************************************************/ + public SavedReport withColumnsJson(String columnsJson) + { + this.columnsJson = columnsJson; + return (this); + } + + + + /******************************************************************************* + ** Getter for inputFieldsJson + *******************************************************************************/ + public String getInputFieldsJson() + { + return (this.inputFieldsJson); + } + + + + /******************************************************************************* + ** Setter for inputFieldsJson + *******************************************************************************/ + public void setInputFieldsJson(String inputFieldsJson) + { + this.inputFieldsJson = inputFieldsJson; + } + + + + /******************************************************************************* + ** Fluent setter for inputFieldsJson + *******************************************************************************/ + public SavedReport withInputFieldsJson(String inputFieldsJson) + { + this.inputFieldsJson = inputFieldsJson; + return (this); + } + + + + /******************************************************************************* + ** Getter for pivotTableJson + *******************************************************************************/ + public String getPivotTableJson() + { + return (this.pivotTableJson); + } + + + + /******************************************************************************* + ** Setter for pivotTableJson + *******************************************************************************/ + public void setPivotTableJson(String pivotTableJson) + { + this.pivotTableJson = pivotTableJson; + } + + + + /******************************************************************************* + ** Fluent setter for pivotTableJson + *******************************************************************************/ + public SavedReport withPivotTableJson(String pivotTableJson) + { + this.pivotTableJson = pivotTableJson; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportJsonFieldDisplayValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportJsonFieldDisplayValueFormatter.java new file mode 100644 index 00000000..cb1af39c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportJsonFieldDisplayValueFormatter.java @@ -0,0 +1,150 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.SavedReportToReportMetaDataAdapter; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedReportJsonFieldDisplayValueFormatter implements FieldDisplayBehavior +{ + private static SavedReportJsonFieldDisplayValueFormatter savedReportJsonFieldDisplayValueFormatter = null; + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private SavedReportJsonFieldDisplayValueFormatter() + { + + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static SavedReportJsonFieldDisplayValueFormatter getInstance() + { + if(savedReportJsonFieldDisplayValueFormatter == null) + { + savedReportJsonFieldDisplayValueFormatter = new SavedReportJsonFieldDisplayValueFormatter(); + } + return (savedReportJsonFieldDisplayValueFormatter); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public SavedReportJsonFieldDisplayValueFormatter getDefault() + { + return getInstance(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) + { + for(QRecord record : CollectionUtils.nonNullList(recordList)) + { + if(field.getName().equals("queryFilterJson")) + { + String queryFilterJson = record.getValueString("queryFilterJson"); + if(StringUtils.hasContent(queryFilterJson)) + { + try + { + QQueryFilter qQueryFilter = SavedReportToReportMetaDataAdapter.getQQueryFilter(queryFilterJson); + int criteriaCount = CollectionUtils.nonNullList(qQueryFilter.getCriteria()).size(); + record.setDisplayValue("queryFilterJson", criteriaCount + " Filter" + StringUtils.plural(criteriaCount)); + } + catch(Exception e) + { + record.setDisplayValue("queryFilterJson", "Invalid Filter..."); + } + } + } + + if(field.getName().equals("columnsJson")) + { + String columnsJson = record.getValueString("columnsJson"); + if(StringUtils.hasContent(columnsJson)) + { + try + { + ReportColumns reportColumns = SavedReportToReportMetaDataAdapter.getReportColumns(columnsJson); + int columnCount = reportColumns.extractVisibleColumns().size(); + + record.setDisplayValue("columnsJson", columnCount + " Column" + StringUtils.plural(columnCount)); + } + catch(Exception e) + { + record.setDisplayValue("columnsJson", "Invalid Columns..."); + } + } + } + + if(field.getName().equals("pivotTableJson")) + { + String pivotTableJson = record.getValueString("pivotTableJson"); + if(StringUtils.hasContent(pivotTableJson)) + { + try + { + PivotTableDefinition pivotTableDefinition = SavedReportToReportMetaDataAdapter.getPivotTableDefinition(pivotTableJson); + int rowCount = CollectionUtils.nonNullList(pivotTableDefinition.getRows()).size(); + int columnCount = CollectionUtils.nonNullList(pivotTableDefinition.getColumns()).size(); + int valueCount = CollectionUtils.nonNullList(pivotTableDefinition.getValues()).size(); + record.setDisplayValue("pivotTableJson", rowCount + " Row" + StringUtils.plural(rowCount) + ", " + columnCount + " Column" + StringUtils.plural(columnCount) + ", and " + valueCount + " Value" + StringUtils.plural(valueCount)); + } + catch(Exception e) + { + record.setDisplayValue("pivotTableJson", "Invalid Pivot Table..."); + } + } + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizer.java new file mode 100644 index 00000000..73679dcf --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizer.java @@ -0,0 +1,282 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.SavedReportToReportMetaDataAdapter; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedReportTableCustomizer implements TableCustomizerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + return (preInsertOrUpdate(records)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + return (preInsertOrUpdate(records)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List preInsertOrUpdate(List records) + { + for(QRecord record : CollectionUtils.nonNullList(records)) + { + preValidateRecord(record); + } + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + void preValidateRecord(QRecord record) + { + try + { + String tableName = record.getValueString("tableName"); + String queryFilterJson = record.getValueString("queryFilterJson"); + String columnsJson = record.getValueString("columnsJson"); + String pivotTableJson = record.getValueString("pivotTableJson"); + + Set usedColumns = new HashSet<>(); + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + if(table == null) + { + record.addError(new BadInputStatusMessage("Unrecognized table name: " + tableName)); + } + + if(StringUtils.hasContent(queryFilterJson)) + { + try + { + //////////////////////////////////////////////////////////////// + // nothing to validate on filter, other than, we can parse it // + //////////////////////////////////////////////////////////////// + SavedReportToReportMetaDataAdapter.getQQueryFilter(queryFilterJson); + } + catch(IOException e) + { + record.addError(new BadInputStatusMessage("Unable to parse queryFilterJson: " + e.getMessage())); + } + } + + boolean hadColumnParseError = false; + if(StringUtils.hasContent(columnsJson)) + { + try + { + ///////////////////////////////////////////////////////////////////////// + // make sure we can parse columns, and that we have at least 1 visible // + ///////////////////////////////////////////////////////////////////////// + ReportColumns reportColumns = SavedReportToReportMetaDataAdapter.getReportColumns(columnsJson); + for(ReportColumn column : reportColumns.extractVisibleColumns()) + { + usedColumns.add(column.getName()); + } + } + catch(IOException e) + { + record.addError(new BadInputStatusMessage("Unable to parse columnsJson: " + e.getMessage())); + hadColumnParseError = true; + } + } + + if(usedColumns.isEmpty() && !hadColumnParseError) + { + record.addError(new BadInputStatusMessage("A Report must contain at least 1 column")); + } + + if(StringUtils.hasContent(pivotTableJson)) + { + try + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure we can parse pivot table, and we have ... at least 1 ... row? maybe that's all that's needed // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + PivotTableDefinition pivotTableDefinition = SavedReportToReportMetaDataAdapter.getPivotTableDefinition(pivotTableJson); + boolean anyRows = false; + boolean missingAnyFieldNamesInRows = false; + boolean missingAnyFieldNamesInColumns = false; + boolean missingAnyFieldNamesInValues = false; + boolean missingAnyFunctionsInValues = false; + + ////////////////// + // look at rows // + ////////////////// + for(PivotTableGroupBy row : CollectionUtils.nonNullList(pivotTableDefinition.getRows())) + { + anyRows = true; + if(StringUtils.hasContent(row.getFieldName())) + { + if(!usedColumns.contains(row.getFieldName()) && !hadColumnParseError) + { + record.addError(new BadInputStatusMessage("A pivot table row is using field (" + getFieldLabelElseName(table, row.getFieldName()) + ") which is not an active column on this report.")); + } + } + else + { + missingAnyFieldNamesInRows = true; + } + } + + if(!anyRows) + { + record.addError(new BadInputStatusMessage("A Pivot Table must contain at least 1 row")); + } + + ///////////////////// + // look at columns // + ///////////////////// + for(PivotTableGroupBy column : CollectionUtils.nonNullList(pivotTableDefinition.getColumns())) + { + if(StringUtils.hasContent(column.getFieldName())) + { + if(!usedColumns.contains(column.getFieldName()) && !hadColumnParseError) + { + record.addError(new BadInputStatusMessage("A pivot table column is using field (" + getFieldLabelElseName(table, column.getFieldName()) + ") which is not an active column on this report.")); + } + } + else + { + missingAnyFieldNamesInColumns = true; + } + } + + //////////////////// + // look at values // + //////////////////// + for(PivotTableValue value : CollectionUtils.nonNullList(pivotTableDefinition.getValues())) + { + if(StringUtils.hasContent(value.getFieldName())) + { + if(!usedColumns.contains(value.getFieldName()) && !hadColumnParseError) + { + record.addError(new BadInputStatusMessage("A pivot table value is using field (" + getFieldLabelElseName(table, value.getFieldName()) + ") which is not an active column on this report.")); + } + } + else + { + missingAnyFieldNamesInValues = true; + } + + if(value.getFunction() == null) + { + missingAnyFunctionsInValues = true; + } + } + + //////////////////////////////////////////////// + // errors based on missing things found above // + //////////////////////////////////////////////// + if(missingAnyFieldNamesInRows) + { + record.addError(new BadInputStatusMessage("Missing field name for at least one pivot table row.")); + } + + if(missingAnyFieldNamesInColumns) + { + record.addError(new BadInputStatusMessage("Missing field name for at least one pivot table column.")); + } + + if(missingAnyFieldNamesInValues) + { + record.addError(new BadInputStatusMessage("Missing field name for at least one pivot table value.")); + } + + if(missingAnyFunctionsInValues) + { + record.addError(new BadInputStatusMessage("Missing function for at least one pivot table value.")); + } + } + catch(IOException e) + { + record.addError(new BadInputStatusMessage("Unable to parse pivotTableJson: " + e.getMessage())); + } + } + } + catch(Exception e) + { + LOG.warn("Error validating a savedReport"); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getFieldLabelElseName(QTableMetaData table, String fieldName) + { + try + { + GenerateReportAction.FieldAndJoinTable fieldAndJoinTable = GenerateReportAction.getFieldAndJoinTable(table, fieldName); + return (fieldAndJoinTable.getLabel(table)); + } + catch(Exception e) + { + return (fieldName); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java new file mode 100644 index 00000000..b82e648f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java @@ -0,0 +1,210 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.util.List; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DefaultWidgetRenderer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; +import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RenderSavedReportMetaDataProducer; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedReportsMetaDataProvider +{ + public static final String REPORT_STORAGE_TABLE_NAME = "reportStorage"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineAll(QInstance instance, String recordTablesBackendName, String reportStorageBackendName, Consumer backendDetailEnricher) throws QException + { + instance.addTable(defineSavedReportTable(recordTablesBackendName, backendDetailEnricher)); + instance.addTable(defineRenderedReportTable(recordTablesBackendName, backendDetailEnricher)); + instance.addPossibleValueSource(QPossibleValueSource.newForTable(SavedReport.TABLE_NAME)); + instance.addPossibleValueSource(QPossibleValueSource.newForEnum(ReportFormatPossibleValueEnum.NAME, ReportFormatPossibleValueEnum.values())); + instance.addPossibleValueSource(QPossibleValueSource.newForEnum(RenderedReportStatus.NAME, RenderedReportStatus.values())); + + instance.addTable(defineReportStorageTable(reportStorageBackendName, backendDetailEnricher)); + + QProcessMetaData renderSavedReportProcess = new RenderSavedReportMetaDataProducer().produce(instance); + instance.addProcess(renderSavedReportProcess); + renderSavedReportProcess.getInputFields().stream() + .filter(f -> RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME.equals(f.getName())) + .findFirst() + .ifPresent(f -> f.setDefaultValue(REPORT_STORAGE_TABLE_NAME)); + + instance.addWidget(defineReportSetupWidget()); + instance.addWidget(definePivotTableSetupWidget()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineReportStorageTable(String backendName, Consumer backendDetailEnricher) + { + QTableMetaData table = new QTableMetaData() + .withName(REPORT_STORAGE_TABLE_NAME) + .withBackendName(backendName) + .withPrimaryKeyField("reference") + .withField(new QFieldMetaData("reference", QFieldType.STRING)) + .withField(new QFieldMetaData("contents", QFieldType.BLOB)); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QWidgetMetaDataInterface defineReportSetupWidget() + { + return new QWidgetMetaData() + .withName("reportSetupWidget") + .withLabel("Filters and Columns") + .withIsCard(true) + .withType(WidgetType.REPORT_SETUP.getType()) + .withCodeReference(new QCodeReference(DefaultWidgetRenderer.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QWidgetMetaDataInterface definePivotTableSetupWidget() + { + return new QWidgetMetaData() + .withName("pivotTableSetupWidget") + .withLabel("Pivot Table") + .withIsCard(true) + .withType(WidgetType.PIVOT_TABLE_SETUP.getType()) + .withCodeReference(new QCodeReference(DefaultWidgetRenderer.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData defineSavedReportTable(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(SavedReport.TABLE_NAME) + .withLabel("Report") + .withIcon(new QIcon().withName("article")) + .withRecordLabelFormat("%s") + .withRecordLabelFields("label") + .withBackendName(backendName) + .withPrimaryKeyField("id") + .withFieldsFromEntity(SavedReport.class) + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "tableName"))) + .withSection(new QFieldSection("filtersAndColumns", new QIcon().withName("table_chart"), Tier.T2).withLabel("Filters and Columns").withWidgetName("reportSetupWidget")) + .withSection(new QFieldSection("pivotTable", new QIcon().withName("pivot_table_chart"), Tier.T2).withLabel("Pivot Table").withWidgetName("pivotTableSetupWidget")) + .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("queryFilterJson", "columnsJson", "pivotTableJson")).withIsHidden(true)) + .withSection(new QFieldSection("hidden", new QIcon().withName("text_snippet"), Tier.T2, List.of("inputFieldsJson", "userId")).withIsHidden(true)) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + table.getField("queryFilterJson").withBehavior(SavedReportJsonFieldDisplayValueFormatter.getInstance()); + table.getField("columnsJson").withBehavior(SavedReportJsonFieldDisplayValueFormatter.getInstance()); + table.getField("pivotTableJson").withBehavior(SavedReportJsonFieldDisplayValueFormatter.getInstance()); + + table.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(SavedReportTableCustomizer.class)); + table.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(SavedReportTableCustomizer.class)); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineRenderedReportTable(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(RenderedReport.TABLE_NAME) + .withIcon(new QIcon().withName("print")) + .withRecordLabelFormat("%s - %s") + .withRecordLabelFields("savedReportId", "startTime") + .withBackendName(backendName) + .withPrimaryKeyField("id") + .withFieldsFromEntity(RenderedReport.class) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedReportId", "renderedReportStatusId"))) + .withSection(new QFieldSection("input", new QIcon().withName("input"), Tier.T2, List.of("userId", "reportFormat"))) + .withSection(new QFieldSection("output", new QIcon().withName("output"), Tier.T2, List.of("jobUuid", "resultPath", "rowCount", "errorMessage", "startTime", "endTime"))) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))) + .withoutCapabilities(Capability.allWriteCapabilities()); + + table.getField("renderedReportStatusId").setAdornments(List.of(new FieldAdornment(AdornmentType.CHIP) + .withValues(AdornmentType.ChipValues.iconAndColorValues(RenderedReportStatus.RUNNING.getId(), "pending", AdornmentType.ChipValues.COLOR_SECONDARY)) + .withValues(AdornmentType.ChipValues.iconAndColorValues(RenderedReportStatus.COMPLETE.getId(), "check", AdornmentType.ChipValues.COLOR_SUCCESS)) + .withValues(AdornmentType.ChipValues.iconAndColorValues(RenderedReportStatus.FAILED.getId(), "error", AdornmentType.ChipValues.COLOR_ERROR)))); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedViewsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedViewsMetaDataProvider.java index 2581e67d..ee6b16da 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedViewsMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedViewsMetaDataProvider.java @@ -68,7 +68,8 @@ public class SavedViewsMetaDataProvider { QTableMetaData table = new QTableMetaData() .withName(SavedView.TABLE_NAME) - .withLabel("Saved View") + .withLabel("View") + .withIcon(new QIcon().withName("table_view")) .withRecordLabelFormat("%s") .withRecordLabelFields("label") .withBackendName(backendName) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java index ac2e313a..0ff5c246 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java @@ -28,6 +28,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -52,9 +53,16 @@ public class QSession implements Serializable private Map> securityKeyValues; private Map backendVariants; - // implementation-specific custom values + /////////////////////////////////////////// + // implementation-specific custom values // + /////////////////////////////////////////// private Map values; + ///////////////////////////////////////////// + // values meant to be passed to a frontend // + ///////////////////////////////////////////// + private Map valuesForFrontend; + public static final String VALUE_KEY_USER_TIMEZONE = "UserTimezone"; public static final String VALUE_KEY_USER_TIMEZONE_OFFSET_MINUTES = "UserTimezoneOffsetMinutes"; @@ -500,4 +508,49 @@ public class QSession implements Serializable return (this); } + + /******************************************************************************* + ** Getter for valuesForFrontend + *******************************************************************************/ + public Map getValuesForFrontend() + { + return (this.valuesForFrontend); + } + + + + /******************************************************************************* + ** Setter for valuesForFrontend + *******************************************************************************/ + public void setValuesForFrontend(Map valuesForFrontend) + { + this.valuesForFrontend = valuesForFrontend; + } + + + + /******************************************************************************* + ** Fluent setter for valuesForFrontend + *******************************************************************************/ + public QSession withValuesForFrontend(Map valuesForFrontend) + { + this.valuesForFrontend = valuesForFrontend; + return (this); + } + + + /******************************************************************************* + ** Fluent setter for a single valuesForFrontend + *******************************************************************************/ + public QSession withValueForFrontend(String key, Serializable value) + { + if(this.valuesForFrontend == null) + { + this.valuesForFrontend = new LinkedHashMap<>(); + } + this.valuesForFrontend.put(key, value); + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleDispatcher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleDispatcher.java index 2b218bd6..d8069f0d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleDispatcher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleDispatcher.java @@ -27,21 +27,18 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; -import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* ** This class is responsible for loading a backend module, by its name, and ** returning an instance. ** - ** TODO - make this mapping runtime-bound, not pre-compiled in. - ** *******************************************************************************/ public class QBackendModuleDispatcher { private static final QLogger LOG = QLogger.getLogger(QBackendModuleDispatcher.class); - private static Map backendTypeToModuleClassNameMap; + private static Map backendTypeToModuleClassNameMap = new HashMap<>(); @@ -50,51 +47,6 @@ public class QBackendModuleDispatcher *******************************************************************************/ public QBackendModuleDispatcher() { - initBackendTypeToModuleClassNameMap(); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private static void initBackendTypeToModuleClassNameMap() - { - if(backendTypeToModuleClassNameMap != null) - { - return; - } - - Map newMap = new HashMap<>(); - - String[] moduleClassNames = new String[] - { - // todo - let modules somehow "export" their types here? - // e.g., backend-core shouldn't need to "know" about the modules. - "com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule", - "com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule", - "com.kingsrook.qqq.backend.core.modules.backend.implementations.enumeration.EnumerationBackendModule", - "com.kingsrook.qqq.backend.module.rdbms.RDBMSBackendModule", - "com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule", - "com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule", - "com.kingsrook.qqq.backend.module.api.APIBackendModule" - }; - - for(String moduleClassName : moduleClassNames) - { - try - { - Class moduleClass = Class.forName(moduleClassName); - QBackendModuleInterface module = (QBackendModuleInterface) moduleClass.getConstructor().newInstance(); - newMap.put(module.getBackendType(), moduleClassName); - } - catch(Exception e) - { - LOG.debug("Backend module could not be loaded", e, logPair("moduleClassName", moduleClassName)); - } - } - - backendTypeToModuleClassNameMap = newMap; } @@ -104,7 +56,6 @@ public class QBackendModuleDispatcher *******************************************************************************/ public static void registerBackendModule(QBackendModuleInterface moduleInstance) { - initBackendTypeToModuleClassNameMap(); String backendType = moduleInstance.getBackendType(); if(backendTypeToModuleClassNameMap.containsKey(backendType)) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleInterface.java index 64ce0c3c..87eef4c5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleInterface.java @@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -39,7 +40,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails /******************************************************************************* ** Interface that a QBackendModule must implement. ** - ** Note, some methods all have a default version, which throws a 'not implemented' + ** Note, all methods have a default version, which throws a 'not implemented' ** exception. ** *******************************************************************************/ @@ -129,6 +130,16 @@ public interface QBackendModuleInterface return null; } + + /******************************************************************************* + ** + *******************************************************************************/ + default QStorageInterface getStorageInterface() + { + throwNotImplemented("StorageInterface"); + return null; + } + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java index 7beb1ec2..53c1c594 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationBackendModule.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.enumerati import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -37,6 +38,10 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; *******************************************************************************/ public class EnumerationBackendModule implements QBackendModuleInterface { + static + { + QBackendModuleDispatcher.registerBackendModule(new EnumerationBackendModule()); + } /******************************************************************************* ** Method where a backend module must be able to provide its type (name). diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java index 4d6a93cb..3203280e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java @@ -26,8 +26,10 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -42,6 +44,12 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; *******************************************************************************/ public class MemoryBackendModule implements QBackendModuleInterface { + static + { + QBackendModuleDispatcher.registerBackendModule(new MemoryBackendModule()); + } + + /******************************************************************************* ** Method where a backend module must be able to provide its type (name). *******************************************************************************/ @@ -117,4 +125,14 @@ public class MemoryBackendModule implements QBackendModuleInterface return (new MemoryDeleteAction()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QStorageInterface getStorageInterface() + { + return (new MemoryStorageAction()); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryStorageAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryStorageAction.java new file mode 100644 index 00000000..313145a7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryStorageAction.java @@ -0,0 +1,149 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory; + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** implementation of bulk-storage interface, for the memory backend module. + ** + ** Requires table to have (at least?) 2 fields - a STRING primary key and a + ** BLOB to store bytes. + *******************************************************************************/ +public class MemoryStorageAction implements QStorageInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public OutputStream createOutputStream(StorageInput storageInput) + { + return new MemoryStorageOutputStream(storageInput.getTableName(), storageInput.getReference()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InputStream getInputStream(StorageInput storageInput) throws QException + { + QRecord record = new GetAction().executeForRecord(new GetInput(storageInput.getTableName()).withPrimaryKey(storageInput.getReference())); + if(record == null) + { + throw (new QNotFoundException("Could not find input stream for [" + storageInput.getTableName() + "][" + storageInput.getReference() + "]")); + } + + QFieldMetaData blobField = getBlobField(storageInput.getTableName()); + return (new ByteArrayInputStream(record.getValueByteArray(blobField.getName()))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QFieldMetaData getBlobField(String tableName) throws QException + { + Optional firstBlobField = QContext.getQInstance().getTable(tableName).getFields().values().stream().filter(f -> QFieldType.BLOB.equals(f.getType())).findFirst(); + if(firstBlobField.isEmpty()) + { + throw (new QException("Could not find a blob field in table [" + tableName + "]")); + } + return firstBlobField.get(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static class MemoryStorageOutputStream extends ByteArrayOutputStream + { + private final String tableName; + private final String reference; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public MemoryStorageOutputStream(String tableName, String reference) + { + this.tableName = tableName; + this.reference = reference; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void close() throws IOException + { + super.close(); + + try + { + QFieldMetaData blobField = getBlobField(tableName); + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(tableName).withRecord(new QRecord() + .withValue(QContext.getQInstance().getTable(tableName).getPrimaryKeyField(), reference) + .withValue(blobField.getName(), toByteArray()))); + + if(CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getErrors())) + { + throw(new IOException("Error storing stream into memory storage: " + StringUtils.joinWithCommasAndAnd(insertOutput.getRecords().get(0).getErrors().stream().map(e -> e.getMessage()).toList()))); + } + } + catch(Exception e) + { + throw new IOException("Wrapped QException", e); + } + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockBackendModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockBackendModule.java index 9ee92353..e530e759 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockBackendModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockBackendModule.java @@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -40,6 +41,11 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; *******************************************************************************/ public class MockBackendModule implements QBackendModuleInterface { + static + { + QBackendModuleDispatcher.registerBackendModule(new MockBackendModule()); + } + /******************************************************************************* ** Method where a backend module must be able to provide its type (name). *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/ExecuteReportStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/ExecuteReportStep.java index 8d71629e..fdfa381c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/ExecuteReportStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/ExecuteReportStep.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; @@ -66,8 +67,9 @@ public class ExecuteReportStep implements BackendStep { ReportInput reportInput = new ReportInput(); reportInput.setReportName(reportName); - reportInput.setReportFormat(ReportFormat.XLSX); // todo - variable - reportInput.setReportOutputStream(reportOutputStream); + reportInput.setReportDestination(new ReportDestination() + .withReportFormat(ReportFormat.XLSX) // todo - variable + .withReportOutputStream(reportOutputStream)); Map values = runBackendStepInput.getValues(); reportInput.setInputValues(values); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java new file mode 100644 index 00000000..9cf6ba45 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java @@ -0,0 +1,180 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.io.OutputStream; +import java.io.Serializable; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Map; +import java.util.UUID; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.savedreports.RenderedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.RenderedReportStatus; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Process step to actually execute rendering a saved report. + *******************************************************************************/ +public class RenderSavedReportExecuteStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(RenderSavedReportExecuteStep.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + QRecord renderedReportRecord = null; + + try + { + //////////////////////////////// + // read inputs, set up params // + //////////////////////////////// + String storageTableName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME); + ReportFormat reportFormat = ReportFormat.fromString(runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT)); + SavedReport savedReport = new SavedReport(runBackendStepInput.getRecords().get(0)); + String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, savedReport); + String storageReference = LocalDate.now() + "/" + LocalTime.now().toString().replaceAll(":", "").replaceFirst("\\..*", "") + "/" + UUID.randomUUID() + "/" + downloadFileBaseName + "." + reportFormat.getExtension(); + OutputStream outputStream = new StorageAction().createOutputStream(new StorageInput(storageTableName).withReference(storageReference)); + + LOG.info("Starting to render a report", logPair("savedReportId", savedReport.getId()), logPair("tableName", savedReport.getTableName()), logPair("storageReference", storageReference)); + runBackendStepInput.getAsyncJobCallback().updateStatus("Generating Report"); + + ////////////////////////////////////////////////////////////////// + // insert a rendered-report record indicating that it's running // + ////////////////////////////////////////////////////////////////// + renderedReportRecord = new InsertAction().execute(new InsertInput(RenderedReport.TABLE_NAME).withRecordEntity(new RenderedReport() + .withSavedReportId(savedReport.getId()) + .withStartTime(Instant.now()) + // todo .withJobUuid(runBackendStepInput.get) + .withRenderedReportStatusId(RenderedReportStatus.RUNNING.getId()) + .withReportFormat(ReportFormatPossibleValueEnum.valueOf(reportFormat.name()).getPossibleValueId()) + )).getRecords().get(0); + + //////////////////////////////////////////////////////////////////////////////////////////// + // convert the report record to report meta-data, which the GenerateReportAction works on // + //////////////////////////////////////////////////////////////////////////////////////////// + QReportMetaData reportMetaData = new SavedReportToReportMetaDataAdapter().adapt(savedReport, reportFormat); + + ///////////////////////////////////// + // setup & run the generate action // + ///////////////////////////////////// + ReportInput reportInput = new ReportInput(); + reportInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); + reportInput.setReportMetaData(reportMetaData); + reportInput.setReportDestination(new ReportDestination() + .withReportFormat(reportFormat) + .withReportOutputStream(outputStream)); + + Map values = runBackendStepInput.getValues(); + reportInput.setInputValues(values); + + ReportOutput reportOutput = new GenerateReportAction().execute(reportInput); + + /////////////////////////////////// + // update record to show success // + /////////////////////////////////// + new UpdateAction().execute(new UpdateInput(RenderedReport.TABLE_NAME).withRecord(new QRecord() + .withValue("id", renderedReportRecord.getValue("id")) + .withValue("resultPath", storageReference) + .withValue("renderedReportStatusId", RenderedReportStatus.COMPLETE.getPossibleValueId()) + .withValue("endTime", Instant.now()) + .withValue("rowCount", reportOutput.getTotalRecordCount()) + )); + + runBackendStepOutput.addValue("downloadFileName", downloadFileBaseName + "." + reportFormat.getExtension()); + runBackendStepOutput.addValue("storageTableName", storageTableName); + runBackendStepOutput.addValue("storageReference", storageReference); + LOG.info("Completed rendering a report", logPair("savedReportId", savedReport.getId()), logPair("tableName", savedReport.getTableName()), logPair("storageReference", storageReference), logPair("rowCount", reportOutput.getTotalRecordCount())); + } + catch(Exception e) + { + if(renderedReportRecord != null) + { + new UpdateAction().execute(new UpdateInput(RenderedReport.TABLE_NAME).withRecord(new QRecord() + .withValue("id", renderedReportRecord.getValue("id")) + .withValue("renderedReportStatusId", RenderedReportStatus.FAILED.getPossibleValueId()) + .withValue("endTime", Instant.now()) + .withValue("errorMessage", ExceptionUtils.concatenateMessagesFromChain(e)) + )); + } + + LOG.warn("Error rendering saved report", e); + throw (e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getDownloadFileBaseName(RunBackendStepInput runBackendStepInput, SavedReport report) + { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HHmm").withZone(ZoneId.systemDefault()); + String datePart = formatter.format(Instant.now()); + + String downloadFileBaseName = runBackendStepInput.getValueString("downloadFileBaseName"); + if(!StringUtils.hasContent(downloadFileBaseName)) + { + downloadFileBaseName = report.getLabel(); + } + + ////////////////////////////////////////////////// + // these chars have caused issues, so, disallow // + ////////////////////////////////////////////////// + downloadFileBaseName = downloadFileBaseName.replaceAll("/", "-").replaceAll(",", "_"); + + return (downloadFileBaseName + " - " + datePart); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java new file mode 100644 index 00000000..f63d914f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; + + +/******************************************************************************* + ** define process for rendering saved reports! + *******************************************************************************/ +public class RenderSavedReportMetaDataProducer implements MetaDataProducerInterface +{ + public static final String NAME = "renderSavedReport"; + + public static final String FIELD_NAME_STORAGE_TABLE_NAME = "storageTableName"; + public static final String FIELD_NAME_REPORT_FORMAT = "reportFormat"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + QProcessMetaData process = new QProcessMetaData() + .withName(NAME) + .withLabel("Render Report") + .withTableName(SavedReport.TABLE_NAME) + .withIcon(new QIcon().withName("print")) + .addStep(new QBackendStepMetaData() + .withName("pre") + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData(FIELD_NAME_STORAGE_TABLE_NAME, QFieldType.STRING)) + .withRecordListMetaData(new QRecordListMetaData().withTableName(SavedReport.TABLE_NAME))) + .withCode(new QCodeReference(RenderSavedReportPreStep.class))) + .addStep(new QFrontendStepMetaData() + .withName("input") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)) + .withFormField(new QFieldMetaData(FIELD_NAME_REPORT_FORMAT, QFieldType.STRING) + .withPossibleValueSourceName(ReportFormatPossibleValueEnum.NAME) + .withIsRequired(true))) + .addStep(new QBackendStepMetaData() + .withName("execute") + .withInputData(new QFunctionInputMetaData().withRecordListMetaData(new QRecordListMetaData() + .withTableName(SavedReport.TABLE_NAME))) + .withCode(new QCodeReference(RenderSavedReportExecuteStep.class))) + .addStep(new QFrontendStepMetaData() + .withName("output") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.DOWNLOAD_FORM))); + + return (process); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportPreStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportPreStep.java new file mode 100644 index 00000000..f74298fb --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportPreStep.java @@ -0,0 +1,79 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** initial backend-step before rendering a saved report. does some basic + ** validations, and then (in future) will set up input fields (how??) for the + ** input screen. + *******************************************************************************/ +public class RenderSavedReportPreStep implements BackendStep +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + String storageTableName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME); + if(!StringUtils.hasContent(storageTableName)) + { + throw (new QUserFacingException("Process configuration error: Missing value for storageTableName.")); + } + + if(QContext.getQInstance().getTable(storageTableName) == null) + { + throw (new QUserFacingException("Process configuration error: Unrecognized value for storageTableName - no table named [" + storageTableName + "] was found in the instance.")); + } + + List records = runBackendStepInput.getRecords(); + if(!CollectionUtils.nullSafeHasContents(records)) + { + throw (new QUserFacingException("No report was selected or found to be rendered.")); + } + + if(records.size() > 1) + { + throw (new QUserFacingException("You may only render 1 report at a time.")); + } + + SavedReport savedReport = new SavedReport(records.get(0)); + + // todo - check for inputs - set up the input screen... + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java new file mode 100644 index 00000000..55ab7270 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportToReportMetaDataAdapter.java @@ -0,0 +1,408 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; +import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumn; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** GenerateReportAction takes in ReportMetaData. + ** + ** This class knows how to adapt from a SavedReport to a ReportMetaData, so that + ** we can render a saved report. + *******************************************************************************/ +public class SavedReportToReportMetaDataAdapter +{ + private static final QLogger LOG = QLogger.getLogger(SavedReportToReportMetaDataAdapter.class); + + private static Consumer jsonMapperCustomizer = om -> om.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QReportMetaData adapt(SavedReport savedReport, ReportFormat reportFormat) throws QException + { + try + { + QInstance qInstance = QContext.getQInstance(); + + QReportMetaData reportMetaData = new QReportMetaData(); + reportMetaData.setName("savedReport:" + savedReport.getId()); + reportMetaData.setLabel(savedReport.getLabel()); + + ///////////////////////////////////////////////////// + // set up the data-source - e.g., table and filter // + ///////////////////////////////////////////////////// + QReportDataSource dataSource = new QReportDataSource(); + reportMetaData.setDataSources(List.of(dataSource)); + dataSource.setName("main"); + + QTableMetaData table = qInstance.getTable(savedReport.getTableName()); + dataSource.setSourceTable(savedReport.getTableName()); + dataSource.setQueryFilter(getQQueryFilter(savedReport.getQueryFilterJson())); + + ////////////////////////// + // set up the main view // + ////////////////////////// + QReportView view = new QReportView(); + reportMetaData.setViews(ListBuilder.of(view)); + view.setName("main"); + view.setType(ReportType.TABLE); + view.setDataSourceName(dataSource.getName()); + view.setLabel(savedReport.getLabel()); + view.setIncludeHeaderRow(true); + + /////////////////////////////////////////////////////////////////////////////////////////////// + // columns in the saved-report should look like a serialized version of ReportColumns object // + // map them to a list of QReportField objects // + // also keep track of what joinTables we find that we need to select // + /////////////////////////////////////////////////////////////////////////////////////////////// + ReportColumns columnsObject = getReportColumns(savedReport.getColumnsJson()); + + List reportColumns = new ArrayList<>(); + view.setColumns(reportColumns); + + Set neededJoinTables = new HashSet<>(); + + for(ReportColumn column : columnsObject.extractVisibleColumns()) + { + //////////////////////////////////////////////////// + // figure out the field being named by the column // + //////////////////////////////////////////////////// + String fieldName = column.getName(); + FieldAndJoinTable fieldAndJoinTable = getField(savedReport, fieldName, qInstance, neededJoinTables, table); + if(fieldAndJoinTable == null) + { + continue; + } + + ////////////////////////////////////////////////// + // make a QReportField based on the table field // + ////////////////////////////////////////////////// + reportColumns.add(makeQReportField(fieldName, fieldAndJoinTable)); + } + + /////////////////////////////////////////////////////////////////////////////////////////// + // set up joins, if we need any // + // note - test coverage here is provided by RDBMS module's GenerateReportActionRDBMSTest // + /////////////////////////////////////////////////////////////////////////////////////////// + if(!neededJoinTables.isEmpty()) + { + List queryJoins = new ArrayList<>(); + dataSource.setQueryJoins(queryJoins); + + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + if(neededJoinTables.contains(exposedJoin.getJoinTable())) + { + QueryJoin queryJoin = new QueryJoin(exposedJoin.getJoinTable()) + .withSelect(true) + .withType(QueryJoin.Type.LEFT) + .withBaseTableOrAlias(null) + .withAlias(null); + + if(exposedJoin.getJoinPath().size() == 1) + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Note, this is similar logic (and comment) in QFMD ... // + // todo - what about a join with a longer path? it would be nice to pass such joinNames through there too, // + // but what, that would actually be multiple queryJoins? needs a fair amount of thought. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + queryJoin.setJoinMetaData(qInstance.getJoin(exposedJoin.getJoinPath().get(0))); + } + + queryJoins.add(queryJoin); + } + } + } + + ///////////////////////////////////////// + // if it's a pivot report, handle that // + ///////////////////////////////////////// + if(StringUtils.hasContent(savedReport.getPivotTableJson())) + { + PivotTableDefinition pivotTableDefinition = getPivotTableDefinition(savedReport.getPivotTableJson()); + + QReportView pivotView = new QReportView(); + reportMetaData.getViews().add(pivotView); + pivotView.setName("pivot"); + pivotView.setLabel("Pivot Table"); + + if(reportFormat != null && reportFormat.getSupportsNativePivotTables()) + { + pivotView.setType(ReportType.PIVOT); + pivotView.setPivotTableSourceViewName(view.getName()); + pivotView.setPivotTableDefinition(pivotTableDefinition); + } + else + { + if(!CollectionUtils.nullSafeHasContents(pivotTableDefinition.getRows())) + { + throw (new QUserFacingException("To generate a pivot report in " + reportFormat + " format, it must have 1 or more Pivot Rows")); + } + + if(CollectionUtils.nullSafeHasContents(pivotTableDefinition.getColumns())) + { + throw (new QUserFacingException("To generate a pivot report in " + reportFormat + " format, it may not have any Pivot Columns")); + } + + /////////////////////// + // handle pivot rows // + /////////////////////// + List summaryFields = new ArrayList<>(); + List summaryOrderByFields = new ArrayList<>(); + for(PivotTableGroupBy row : pivotTableDefinition.getRows()) + { + String fieldName = row.getFieldName(); + FieldAndJoinTable fieldAndJoinTable = getField(savedReport, fieldName, qInstance, neededJoinTables, table); + if(fieldAndJoinTable == null) + { + LOG.warn("The field for a Pivot Row wasn't found, when converting to a summary...", logPair("savedReportId", savedReport.getId()), logPair("fieldName", fieldName)); + continue; + } + summaryFields.add(fieldName); + summaryOrderByFields.add(new QFilterOrderBy(fieldName)); + } + + ///////////////////////// + // handle pivot values // + ///////////////////////// + List summaryViewColumns = new ArrayList<>(); + for(PivotTableValue value : pivotTableDefinition.getValues()) + { + String fieldName = value.getFieldName(); + FieldAndJoinTable fieldAndJoinTable = getField(savedReport, fieldName, qInstance, neededJoinTables, table); + if(fieldAndJoinTable == null) + { + LOG.warn("The field for a Pivot Value wasn't found, when converting to a summary...", logPair("savedReportId", savedReport.getId()), logPair("fieldName", fieldName)); + continue; + } + + QReportField reportField = makeQReportField(fieldName, fieldAndJoinTable); + reportField.setName(fieldName + "_" + value.getFunction().name()); + reportField.setLabel(StringUtils.ucFirst(value.getFunction().name().toLowerCase()) + " Of " + reportField.getLabel()); + reportField.setFormula("${pivot." + value.getFunction().name().toLowerCase() + "." + fieldName + "}"); + summaryViewColumns.add(reportField); + summaryOrderByFields.add(new QFilterOrderBy(reportField.getName())); + } + + pivotView.setType(ReportType.SUMMARY); + pivotView.setDataSourceName(dataSource.getName()); + pivotView.setIncludeHeaderRow(true); + pivotView.setIncludeTotalRow(true); + pivotView.setColumns(summaryViewColumns); + pivotView.setSummaryFields(summaryFields); + pivotView.withOrderByFields(summaryOrderByFields); + } + + //////////////////////////////////////////////////////////////////////////////////// + // in case the reportFormat doesn't support multiple views, and we have a pivot - // + // then remove the data view // + //////////////////////////////////////////////////////////////////////////////////// + if(reportFormat != null && !reportFormat.getSupportsMultipleViews()) + { + reportMetaData.getViews().remove(0); + } + } + + ///////////////////////////////////////////////////// + // add input fields, if they're in the savedReport // + ///////////////////////////////////////////////////// + if(StringUtils.hasContent(savedReport.getInputFieldsJson())) + { + //////////////////////////////////// + // todo turn on when implementing // + //////////////////////////////////// + // reportMetaData.setInputFields(JsonUtils.toObject(savedReport.getInputFieldsJson(), new TypeReference<>() {}), objectMapperConsumer); + throw (new IllegalStateException("Input Fields are not yet implemented")); + } + + return (reportMetaData); + } + catch(Exception e) + { + LOG.warn("Error adapting savedReport to reportMetaData", e, logPair("savedReportId", savedReport.getId())); + throw (new QException("Error adapting savedReport to reportMetaData", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static PivotTableDefinition getPivotTableDefinition(String pivotTableJson) throws IOException + { + return JsonUtils.toObject(pivotTableJson, PivotTableDefinition.class, jsonMapperCustomizer); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ReportColumns getReportColumns(String columnsJson) throws IOException + { + return JsonUtils.toObject(columnsJson, ReportColumns.class, jsonMapperCustomizer); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QQueryFilter getQQueryFilter(String queryFilterJson) throws IOException + { + return JsonUtils.toObject(queryFilterJson, QQueryFilter.class, jsonMapperCustomizer); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QReportField makeQReportField(String fieldName, FieldAndJoinTable fieldAndJoinTable) + { + QReportField reportField = new QReportField(); + + reportField.setName(fieldName); + + if(fieldAndJoinTable.joinTable() == null) + { + //////////////////////////////////////////////////////////// + // for fields from this table, just use the field's label // + //////////////////////////////////////////////////////////// + reportField.setLabel(fieldAndJoinTable.field().getLabel()); + } + else + { + /////////////////////////////////////////////////////////////// + // for fields from join tables, use table label: field label // + /////////////////////////////////////////////////////////////// + reportField.setLabel(fieldAndJoinTable.joinTable().getLabel() + ": " + fieldAndJoinTable.field().getLabel()); + } + + if(StringUtils.hasContent(fieldAndJoinTable.field().getPossibleValueSourceName())) + { + reportField.setShowPossibleValueLabel(true); + } + + reportField.setType(fieldAndJoinTable.field().getType()); + + return reportField; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static FieldAndJoinTable getField(SavedReport savedReport, String fieldName, QInstance qInstance, Set neededJoinTables, QTableMetaData table) + { + QFieldMetaData field; + if(fieldName.contains(".")) + { + String joinTableName = fieldName.replaceAll("\\..*", ""); + String joinFieldName = fieldName.replaceAll(".*\\.", ""); + + QTableMetaData joinTable = qInstance.getTable(joinTableName); + if(joinTable == null) + { + LOG.warn("Saved Report has an unrecognized join table name", logPair("savedReportId", savedReport.getId()), logPair("joinTable", joinTable), logPair("fieldName", fieldName)); + return null; + } + + neededJoinTables.add(joinTableName); + + field = joinTable.getFields().get(joinFieldName); + if(field == null) + { + LOG.warn("Saved Report has an unrecognized join field name", logPair("savedReportId", savedReport.getId()), logPair("fieldName", fieldName)); + return null; + } + + return new FieldAndJoinTable(field, joinTable); + } + else + { + field = table.getFields().get(fieldName); + if(field == null) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // frontend may often pass __checked__ (or maybe other __ prefixes in the future - so - don't warn that. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(!fieldName.startsWith("__")) + { + LOG.warn("Saved Report has an unexpected unrecognized field name", logPair("savedReportId", savedReport.getId()), logPair("table", table.getName()), logPair("fieldName", fieldName)); + } + return null; + } + + return new FieldAndJoinTable(field, null); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable) {} +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index 55734984..a2b83ac6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -77,14 +77,14 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt private ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine(); private ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO) - .withMessageSuffix("because of this process' configuration.") + .withMessageSuffix("because this process is not configured to insert records.") .withSingularFutureMessage("will not be inserted ") .withPluralFutureMessage("will not be inserted ") .withSingularPastMessage("was not inserted ") .withPluralPastMessage("were not inserted "); private ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO) - .withMessageSuffix("because of this process' configuration.") + .withMessageSuffix("because this process is not configured to update records.") .withSingularFutureMessage("will not be updated ") .withPluralFutureMessage("will not be updated ") .withSingularPastMessage("was not updated ") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index a51999ab..3dfec3ae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -191,6 +191,41 @@ public class QScheduleManager + /******************************************************************************* + ** + *******************************************************************************/ + public void setupAllNewSchedules() throws QException + { + if(QContext.getQInstance().getTables().containsKey(ScheduledJob.TABLE_NAME)) + { + List scheduledJobList = new QueryAction() + .execute(new QueryInput(ScheduledJob.TABLE_NAME) + .withIncludeAssociations(true)) + .getRecordEntities(ScheduledJob.class); + + for(ScheduledJob scheduledJob : scheduledJobList) + { + try + { + QSchedulerInterface scheduler = getScheduler(scheduledJob.getSchedulerName()); + BasicSchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(scheduledJob); + SchedulableType schedulableType = qInstance.getSchedulableType(scheduledJob.getType()); + + if(!scheduler.isScheduled(schedulableIdentity, schedulableType)) + { + setupScheduledJob(scheduledJob); + } + } + catch(Exception e) + { + LOG.warn("Error evaluating scheduled job", logPair("id", scheduledJob.getId())); + } + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java index b9f16902..3d99ee0b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java @@ -27,6 +27,7 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; @@ -65,6 +66,11 @@ public interface QSchedulerInterface *******************************************************************************/ void unscheduleSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType); + /******************************************************************************* + ** + *******************************************************************************/ + boolean isScheduled(BasicSchedulableIdentity schedulableIdentity, SchedulableType schedulableType); + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcess.java new file mode 100644 index 00000000..408ec1b8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcess.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; + + +/******************************************************************************* + ** Management process to schedule all new scheduled jobs (in all schedulers). + *******************************************************************************/ +public class ScheduleAllNewJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Schedule all New Scheduled Jobs") + .withIcon(new QIcon("more_time")) + .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to schedule all new jobs."))), + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All new jobs have been scheduled."))))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QScheduleManager.getInstance().setupAllNewSchedules(); + } + catch(Exception e) + { + throw (new QException("Error scheduling new jobs.", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java index da5002c8..a9bb5434 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java @@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaDa import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.memoization.AnyKey; @@ -486,6 +487,26 @@ public class QuartzScheduler implements QSchedulerInterface + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean isScheduled(BasicSchedulableIdentity schedulableIdentity, SchedulableType schedulableType) + { + try + { + JobKey jobKey = new JobKey(schedulableIdentity.getIdentity(), schedulableType.getName()); + return (isJobAlreadyScheduled(jobKey)); + } + catch(Exception e) + { + LOG.warn("Error checking if job is scheduled", logPair("identity", schedulableIdentity)); + return (false); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java index 04ff9a6f..8574fecc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java @@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaDa import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -203,6 +204,17 @@ public class SimpleScheduler implements QSchedulerInterface + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean isScheduled(BasicSchedulableIdentity schedulableIdentity, SchedulableType schedulableType) + { + return (executors.containsKey(schedulableIdentity)); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java index e63a43ad..52d756ec 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java @@ -46,8 +46,8 @@ public class InMemoryStateProvider implements StateProviderInterface private final Map map; - private static int jobPeriodSeconds = 60 * 60; // 1 hour - private static int cleanHours = 6; + private static int jobPeriodSeconds = 60 * 30; // 30 minutes + private static int cleanHours = 5; private static int jobInitialDelay = 60 * 60 * cleanHours; @@ -151,6 +151,18 @@ public class InMemoryStateProvider implements StateProviderInterface + /******************************************************************************* + ** Get the current status + * + *******************************************************************************/ + @Override + public String status() + { + return ("InMemoryStateProvider map size: " + map.size()); + } + + + /******************************************************************************* ** Clean entries that started before the given Instant * diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java index c08558d5..43794edd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java @@ -59,6 +59,11 @@ public interface StateProviderInterface *******************************************************************************/ void remove(AbstractStateKey key); + /******************************************************************************* + ** Get the current status + *******************************************************************************/ + String status(); + /******************************************************************************* ** Clean entries that started before the given Instant *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java index 02ab06a9..7f091f1a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java @@ -26,6 +26,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.Serializable; +import java.nio.file.NoSuchFileException; import java.time.Instant; import java.util.Optional; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -99,14 +100,14 @@ public class TempFileStateProvider implements StateProviderInterface String json = FileUtils.readFileToString(getFile(key)); return (Optional.of(JsonUtils.toObject(json, type))); } - catch(FileNotFoundException fnfe) + catch(FileNotFoundException | NoSuchFileException fnfe) { return (Optional.empty()); } catch(IOException e) { LOG.error("Error getting state from file", e); - throw (new RuntimeException("Error retreiving state", e)); + throw (new RuntimeException("Error retrieving state", e)); } } @@ -127,6 +128,18 @@ public class TempFileStateProvider implements StateProviderInterface + /******************************************************************************* + ** Get the current status + * + *******************************************************************************/ + @Override + public String status() + { + return ("TempFileStateProvider status not supported"); + } + + + /******************************************************************************* ** Clean entries that started before the given Instant *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/LocalMacDevUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/LocalMacDevUtils.java new file mode 100644 index 00000000..d2725045 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/LocalMacDevUtils.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils; + + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import com.kingsrook.qqq.backend.core.logging.QLogger; + + +/******************************************************************************* + ** Useful things to do on a mac, when doing development - that we can expect + ** may not exist in a prod or even CI environment. So, they'll only happen if + ** flags are set to do them, and if we're on a mac (e.g., paths exist) + *******************************************************************************/ +public class LocalMacDevUtils +{ + private static final QLogger LOG = QLogger.getLogger(LocalMacDevUtils.class); + + public static boolean mayOpenFiles = false; + + private static final String OPEN_PROGRAM_PATH = "/usr/bin/open"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void openFile(String path) throws IOException + { + if(mayOpenFiles && Files.exists(Path.of(OPEN_PROGRAM_PATH))) + { + Runtime.getRuntime().exec(new String[] { OPEN_PROGRAM_PATH, path }); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void openFile(String path, String appPath) throws IOException + { + if(mayOpenFiles && Files.exists(Path.of(OPEN_PROGRAM_PATH))) + { + if(Files.exists(Path.of(appPath))) + { + Runtime.getRuntime().exec(new String[] { OPEN_PROGRAM_PATH, "-a", appPath, path }); + } + else + { + LOG.warn("App at path [" + appPath + " was not found - file [" + path + "] will not be opened."); + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ObjectUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ObjectUtils.java index 80901c44..0fa1566e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ObjectUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ObjectUtils.java @@ -139,4 +139,28 @@ public class ObjectUtils return (b); } + + + /******************************************************************************* + ** Utility to test a chained unsafe expression CAN get to the end and return true. + ** + ** e.g., instead of: + ** if(a && a.b && a.b.c && a.b.c.d) + ** we can do: + ** if(ifCan(() -> a.b.c.d)) + ** + ** Note - if the supplier returns null, that counts as false! + *******************************************************************************/ + public static boolean ifCan(UnsafeSupplier supplier) + { + try + { + return supplier.get(); + } + catch(Throwable t) + { + return (false); + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesInterface.java index 074c2469..ee6b0a59 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesInterface.java @@ -29,8 +29,12 @@ import java.math.BigDecimal; /******************************************************************************* ** Classes that support doing data aggregations (e.g., count, sum, min, max, average). ** Sub-classes should supply the type parameter. + ** + ** The AVG_T parameter describes the type used for the average getAverage method + ** which, e.g, for date types, might be a date, vs. numbers, they'd probably be + ** BigDecimal. *******************************************************************************/ -public interface AggregatesInterface +public interface AggregatesInterface { /******************************************************************************* ** @@ -60,5 +64,51 @@ public interface AggregatesInterface /******************************************************************************* ** *******************************************************************************/ - BigDecimal getAverage(); + AVG_T getAverage(); + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getProduct() + { + return (null); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getVariance() + { + return (null); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getVarP() + { + return (null); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getStandardDeviation() + { + return (null); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default BigDecimal getStdDevP() + { + return (null); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/BigDecimalAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/BigDecimalAggregates.java index da7f1703..76a6f0d8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/BigDecimalAggregates.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/BigDecimalAggregates.java @@ -28,13 +28,16 @@ import java.math.BigDecimal; /******************************************************************************* ** BigDecimal version of data aggregator *******************************************************************************/ -public class BigDecimalAggregates implements AggregatesInterface +public class BigDecimalAggregates implements AggregatesInterface { private int count = 0; // private Integer countDistinct; private BigDecimal sum; private BigDecimal min; private BigDecimal max; + private BigDecimal product; + + private VarianceCalculator varianceCalculator = new VarianceCalculator(); @@ -59,6 +62,15 @@ public class BigDecimalAggregates implements AggregatesInterface sum = sum.add(input); } + if(product == null) + { + product = input; + } + else + { + product = product.multiply(input); + } + if(min == null || input.compareTo(min) < 0) { min = input; @@ -68,6 +80,52 @@ public class BigDecimalAggregates implements AggregatesInterface { max = input; } + + varianceCalculator.updateVariance(input); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVariance() + { + return (varianceCalculator.getVariance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVarP() + { + return (varianceCalculator.getVarP()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStandardDeviation() + { + return (varianceCalculator.getStandardDeviation()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStdDevP() + { + return (varianceCalculator.getStdDevP()); } @@ -116,6 +174,18 @@ public class BigDecimalAggregates implements AggregatesInterface + /******************************************************************************* + ** Getter for product + ** + *******************************************************************************/ + @Override + public BigDecimal getProduct() + { + return product; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/InstantAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/InstantAggregates.java new file mode 100644 index 00000000..adb1a591 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/InstantAggregates.java @@ -0,0 +1,136 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +import java.math.BigInteger; +import java.time.Instant; + + +/******************************************************************************* + ** Instant version of data aggregator + *******************************************************************************/ +public class InstantAggregates implements AggregatesInterface +{ + private int count = 0; + // private Integer countDistinct; + + private BigInteger sumMillis = BigInteger.ZERO; + + private Instant min; + private Instant max; + + + + /******************************************************************************* + ** Add a new value to this aggregate set + *******************************************************************************/ + public void add(Instant input) + { + if(input == null) + { + return; + } + + count++; + + sumMillis = sumMillis.add(new BigInteger(String.valueOf(input.toEpochMilli()))); + + if(min == null || input.compareTo(min) < 0) + { + min = input; + } + + if(max == null || input.compareTo(max) > 0) + { + max = input; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int getCount() + { + return (count); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Instant getSum() + { + ////////////////////////////////////////// + // sum of date-times doesn't make sense // + ////////////////////////////////////////// + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Instant getMin() + { + return (min); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Instant getMax() + { + return (max); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Instant getAverage() + { + if(this.count > 0) + { + BigInteger averageMillis = this.sumMillis.divide(new BigInteger(String.valueOf(count))); + if(averageMillis.compareTo(new BigInteger(String.valueOf(Long.MAX_VALUE))) < 0) + { + return (Instant.ofEpochMilli(averageMillis.longValue())); + } + } + + return (null); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/IntegerAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/IntegerAggregates.java index 292e8a01..15efecea 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/IntegerAggregates.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/IntegerAggregates.java @@ -28,13 +28,16 @@ import java.math.BigDecimal; /******************************************************************************* ** Integer version of data aggregator *******************************************************************************/ -public class IntegerAggregates implements AggregatesInterface +public class IntegerAggregates implements AggregatesInterface { - private int count = 0; + private int count = 0; // private Integer countDistinct; - private Integer sum; - private Integer min; - private Integer max; + private Integer sum; + private Integer min; + private Integer max; + private BigDecimal product; + + private VarianceCalculator varianceCalculator = new VarianceCalculator(); @@ -48,6 +51,8 @@ public class IntegerAggregates implements AggregatesInterface return; } + BigDecimal inputBD = new BigDecimal(input); + count++; if(sum == null) @@ -59,6 +64,15 @@ public class IntegerAggregates implements AggregatesInterface sum = sum + input; } + if(product == null) + { + product = inputBD; + } + else + { + product = product.multiply(inputBD); + } + if(min == null || input < min) { min = input; @@ -68,6 +82,52 @@ public class IntegerAggregates implements AggregatesInterface { max = input; } + + varianceCalculator.updateVariance(inputBD); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVariance() + { + return (varianceCalculator.getVariance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVarP() + { + return (varianceCalculator.getVarP()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStandardDeviation() + { + return (varianceCalculator.getStandardDeviation()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStdDevP() + { + return (varianceCalculator.getStdDevP()); } @@ -116,6 +176,18 @@ public class IntegerAggregates implements AggregatesInterface + /******************************************************************************* + ** Getter for product + ** + *******************************************************************************/ + @Override + public BigDecimal getProduct() + { + return product; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LocalDateAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LocalDateAggregates.java new file mode 100644 index 00000000..3c64e200 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LocalDateAggregates.java @@ -0,0 +1,136 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +import java.math.BigInteger; +import java.time.LocalDate; + + +/******************************************************************************* + ** LocalDate version of data aggregator + *******************************************************************************/ +public class LocalDateAggregates implements AggregatesInterface +{ + private int count = 0; + // private Integer countDistinct; + + private BigInteger sumMillis = BigInteger.ZERO; + + private LocalDate min; + private LocalDate max; + + + + /******************************************************************************* + ** Add a new value to this aggregate set + *******************************************************************************/ + public void add(LocalDate input) + { + if(input == null) + { + return; + } + + count++; + + sumMillis = sumMillis.add(new BigInteger(String.valueOf(input.toEpochDay()))); + + if(min == null || input.compareTo(min) < 0) + { + min = input; + } + + if(max == null || input.compareTo(max) > 0) + { + max = input; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int getCount() + { + return (count); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public LocalDate getSum() + { + ////////////////////////////////////////// + // sum of date-times doesn't make sense // + ////////////////////////////////////////// + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public LocalDate getMin() + { + return (min); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public LocalDate getMax() + { + return (max); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public LocalDate getAverage() + { + if(this.count > 0) + { + BigInteger averageEpochDay = this.sumMillis.divide(new BigInteger(String.valueOf(count))); + if(averageEpochDay.compareTo(new BigInteger(String.valueOf(Long.MAX_VALUE))) < 0) + { + return (LocalDate.ofEpochDay(averageEpochDay.longValue())); + } + } + + return (null); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java index bcf1862b..e131cda1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java @@ -28,13 +28,16 @@ import java.math.BigDecimal; /******************************************************************************* ** Long version of data aggregator *******************************************************************************/ -public class LongAggregates implements AggregatesInterface +public class LongAggregates implements AggregatesInterface { private int count = 0; // private Long countDistinct; private Long sum; private Long min; private Long max; + private BigDecimal product; + + private VarianceCalculator varianceCalculator = new VarianceCalculator(); @@ -48,6 +51,8 @@ public class LongAggregates implements AggregatesInterface return; } + BigDecimal inputBD = new BigDecimal(input); + count++; if(sum == null) @@ -59,6 +64,15 @@ public class LongAggregates implements AggregatesInterface sum = sum + input; } + if(product == null) + { + product = inputBD; + } + else + { + product = product.multiply(inputBD); + } + if(min == null || input < min) { min = input; @@ -68,6 +82,52 @@ public class LongAggregates implements AggregatesInterface { max = input; } + + varianceCalculator.updateVariance(inputBD); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVariance() + { + return (varianceCalculator.getVariance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getVarP() + { + return (varianceCalculator.getVarP()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStandardDeviation() + { + return (varianceCalculator.getStandardDeviation()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getStdDevP() + { + return (varianceCalculator.getStdDevP()); } @@ -116,6 +176,18 @@ public class LongAggregates implements AggregatesInterface + /******************************************************************************* + ** Getter for product + ** + *******************************************************************************/ + @Override + public BigDecimal getProduct() + { + return product; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/StringAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/StringAggregates.java new file mode 100644 index 00000000..33e306f1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/StringAggregates.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +/******************************************************************************* + ** String version of data aggregator + *******************************************************************************/ +public class StringAggregates implements AggregatesInterface +{ + private int count = 0; + + private String min; + private String max; + + + + /******************************************************************************* + ** Add a new value to this aggregate set + *******************************************************************************/ + public void add(String input) + { + if(input == null) + { + return; + } + + count++; + + if(min == null || input.compareTo(min) < 0) + { + min = input; + } + + if(max == null || input.compareTo(max) > 0) + { + max = input; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int getCount() + { + return (count); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getSum() + { + ////////////////////////////////////// + // sum of string doesn't make sense // + ////////////////////////////////////// + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getMin() + { + return (min); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getMax() + { + return (max); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getAverage() + { + /////////////////////////////////////// + // average string doesn't make sense // + /////////////////////////////////////// + return (null); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/VarianceCalculator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/VarianceCalculator.java new file mode 100644 index 00000000..eefe04f6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/VarianceCalculator.java @@ -0,0 +1,117 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +import java.math.BigDecimal; +import java.math.RoundingMode; + + +/******************************************************************************* + ** see https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm + ** + *******************************************************************************/ +public class VarianceCalculator +{ + private int n; + private BigDecimal runningMean = BigDecimal.ZERO; + private BigDecimal m2 = BigDecimal.ZERO; + + public static int scaleForVarianceCalculations = 4; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void updateVariance(BigDecimal newInput) + { + n++; + BigDecimal delta = newInput.subtract(runningMean); + runningMean = runningMean.add(delta.divide(new BigDecimal(n), scaleForVarianceCalculations, RoundingMode.HALF_UP)); + BigDecimal delta2 = newInput.subtract(runningMean); + m2 = m2.add(delta.multiply(delta2)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BigDecimal getVariance() + { + if(n < 2) + { + return (null); + } + + return m2.divide(new BigDecimal(n - 1), scaleForVarianceCalculations, RoundingMode.HALF_UP); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BigDecimal getVarP() + { + if(n < 2) + { + return (null); + } + + return m2.divide(new BigDecimal(n), scaleForVarianceCalculations, RoundingMode.HALF_UP); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BigDecimal getStandardDeviation() + { + BigDecimal variance = getVariance(); + if(variance == null) + { + return (null); + } + + return BigDecimal.valueOf(Math.sqrt(variance.doubleValue())).setScale(scaleForVarianceCalculations, RoundingMode.HALF_UP); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BigDecimal getStdDevP() + { + BigDecimal varP = getVarP(); + if(varP == null) + { + return (null); + } + + return BigDecimal.valueOf(Math.sqrt(varP.doubleValue())).setScale(scaleForVarianceCalculations, RoundingMode.HALF_UP); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java index 9ff5f7de..ff089474 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportActionTest.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -43,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.LocalMacDevUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -118,6 +120,26 @@ class ExportActionTest extends BaseTest runReport(recordCount, filename, ReportFormat.XLSX, true); File file = new File(filename); + LocalMacDevUtils.openFile(file.getAbsolutePath()); + + assertTrue(file.delete()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testExcelPOI() throws Exception + { + int recordCount = 1000; + String filename = "/tmp/ReportActionTest-POI.xlsx"; + + runReport(recordCount, filename, ReportFormat.XLSX, true); + + File file = new File(filename); + LocalMacDevUtils.openFile(file.getAbsolutePath()); assertTrue(file.delete()); } @@ -147,9 +169,10 @@ class ExportActionTest extends BaseTest ExportInput exportInput = new ExportInput(); exportInput.setTableName(TestUtils.TABLE_NAME_ORDER); - exportInput.setReportFormat(ReportFormat.CSV); ByteArrayOutputStream reportOutputStream = new ByteArrayOutputStream(); - exportInput.setReportOutputStream(reportOutputStream); + exportInput.setReportDestination(new ReportDestination() + .withReportFormat(ReportFormat.CSV) + .withReportOutputStream(reportOutputStream)); exportInput.setQueryFilter(new QQueryFilter()); exportInput.setFieldNames(List.of("id", "orderNo", "storeId", "orderLine.id", "orderLine.sku", "orderLine.quantity")); // exportInput.setFieldNames(List.of("id", "orderNo", "storeId")); @@ -197,8 +220,7 @@ class ExportActionTest extends BaseTest exportInput.setTableName("person"); QTableMetaData table = exportInput.getTable(); - exportInput.setReportFormat(reportFormat); - exportInput.setReportOutputStream(outputStream); + exportInput.setReportDestination(new ReportDestination().withReportFormat(reportFormat).withReportOutputStream(outputStream)); exportInput.setQueryFilter(new QQueryFilter()); exportInput.setLimit(recordCount); @@ -243,7 +265,7 @@ class ExportActionTest extends BaseTest /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // use xlsx, which has a max-rows limit, to verify that code runs, but doesn't throw when there aren't too many rows // /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - exportInput.setReportFormat(ReportFormat.XLSX); + exportInput.setReportDestination(new ReportDestination().withReportFormat(ReportFormat.XLSX)); new ExportAction().preExecute(exportInput); @@ -278,7 +300,7 @@ class ExportActionTest extends BaseTest //////////////////////////////////////////////////////////////// // use xlsx, which has a max-cols limit, to verify that code. // //////////////////////////////////////////////////////////////// - exportInput.setReportFormat(ReportFormat.XLSX); + exportInput.setReportDestination(new ReportDestination().withReportFormat(ReportFormat.XLSX)); assertThrows(QUserFacingException.class, () -> { diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java index 590d96e6..5f354631 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java @@ -23,19 +23,33 @@ package com.kingsrook.qqq.backend.core.actions.reporting; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.Serializable; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.Month; +import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel.ExcelFastexcelExportStreamer; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.poi.BoldHeaderAndFooterPoiExcelStyler; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.poi.ExcelPoiBasedStreamingExportStreamer; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.poi.PoiExcelStylerInterface; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableFunction; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; @@ -51,7 +65,17 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.testutils.PersonQRecord; +import com.kingsrook.qqq.backend.core.utils.LocalMacDevUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.apache.commons.io.FileUtils; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.usermodel.XSSFPivotTable; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.json.JSONArray; +import org.json.JSONObject; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -85,14 +109,14 @@ public class GenerateReportActionTest extends BaseTest ** *******************************************************************************/ @Test - void testPivot1() throws QException + void testSummary1() throws QException { QInstance qInstance = QContext.getQInstance(); - qInstance.addReport(definePersonShoesPivotReport(true)); + qInstance.addReport(definePersonShoesSummaryReport(true)); insertPersonRecords(qInstance); runReport(qInstance, Map.of("startDate", LocalDate.of(1980, Month.JANUARY, 1), "endDate", LocalDate.of(1980, Month.DECEMBER, 31))); - List> list = ListOfMapsExportStreamer.getList("pivot"); + List> list = ListOfMapsExportStreamer.getList("summary"); Iterator> iterator = list.iterator(); Map row = iterator.next(); assertEquals(3, list.size()); @@ -140,10 +164,10 @@ public class GenerateReportActionTest extends BaseTest ** *******************************************************************************/ @Test - void testPivot2() throws QException + void testSummary2() throws QException { QInstance qInstance = QContext.getQInstance(); - QReportMetaData report = definePersonShoesPivotReport(false); + QReportMetaData report = definePersonShoesSummaryReport(false); ////////////////////////////////////////////// // change from the default to sort reversed // @@ -153,7 +177,7 @@ public class GenerateReportActionTest extends BaseTest insertPersonRecords(qInstance); runReport(qInstance, Map.of("startDate", LocalDate.of(1980, Month.JANUARY, 1), "endDate", LocalDate.of(1980, Month.DECEMBER, 31))); - List> list = ListOfMapsExportStreamer.getList("pivot"); + List> list = ListOfMapsExportStreamer.getList("summary"); Iterator> iterator = list.iterator(); Map row = iterator.next(); assertEquals(2, list.size()); @@ -172,10 +196,10 @@ public class GenerateReportActionTest extends BaseTest ** *******************************************************************************/ @Test - void testPivot3() throws QException + void testSummary3() throws QException { QInstance qInstance = QContext.getQInstance(); - QReportMetaData report = definePersonShoesPivotReport(false); + QReportMetaData report = definePersonShoesSummaryReport(false); ////////////////////////////////////////////////////////////////////////////////////////////// // remove the filters, change to sort by personCount (to get some ties), then sumPrice desc // @@ -187,7 +211,7 @@ public class GenerateReportActionTest extends BaseTest insertPersonRecords(qInstance); runReport(qInstance, Map.of("startDate", LocalDate.now(), "endDate", LocalDate.now())); - List> list = ListOfMapsExportStreamer.getList("pivot"); + List> list = ListOfMapsExportStreamer.getList("summary"); Iterator> iterator = list.iterator(); Map row = iterator.next(); @@ -224,16 +248,16 @@ public class GenerateReportActionTest extends BaseTest ** *******************************************************************************/ @Test - void testPivot4() throws QException + void testSummary4() throws QException { QInstance qInstance = QContext.getQInstance(); - QReportMetaData report = definePersonShoesPivotReport(false); + QReportMetaData report = definePersonShoesSummaryReport(false); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // remove the filter, change to have 2 pivot columns - homeStateId and lastName - we should get no roll-up like this. // //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// report.getDataSources().get(0).getQueryFilter().setCriteria(null); - report.getViews().get(0).setPivotFields(List.of( + report.getViews().get(0).setSummaryFields(List.of( "homeStateId", "lastName" )); @@ -241,7 +265,7 @@ public class GenerateReportActionTest extends BaseTest insertPersonRecords(qInstance); runReport(qInstance, Map.of("startDate", LocalDate.now(), "endDate", LocalDate.now())); - List> list = ListOfMapsExportStreamer.getList("pivot"); + List> list = ListOfMapsExportStreamer.getList("summary"); Iterator> iterator = list.iterator(); Map row = iterator.next(); assertEquals(6, list.size()); @@ -282,21 +306,21 @@ public class GenerateReportActionTest extends BaseTest ** *******************************************************************************/ @Test - void testPivot5() throws QException + void testSummary5() throws QException { QInstance qInstance = QContext.getQInstance(); - QReportMetaData report = definePersonShoesPivotReport(false); + QReportMetaData report = definePersonShoesSummaryReport(false); ///////////////////////////////////////////////////////////////////////////////////// // remove the filter, and just pivot on homeStateId - should aggregate differently // ///////////////////////////////////////////////////////////////////////////////////// report.getDataSources().get(0).getQueryFilter().setCriteria(null); - report.getViews().get(0).setPivotFields(List.of("homeStateId")); + report.getViews().get(0).setSummaryFields(List.of("homeStateId")); qInstance.addReport(report); insertPersonRecords(qInstance); runReport(qInstance, Map.of("startDate", LocalDate.now(), "endDate", LocalDate.now())); - List> list = ListOfMapsExportStreamer.getList("pivot"); + List> list = ListOfMapsExportStreamer.getList("summary"); Iterator> iterator = list.iterator(); Map row = iterator.next(); assertEquals(2, list.size()); @@ -315,23 +339,18 @@ public class GenerateReportActionTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - @Test - void runToCsv() throws Exception + private String runToString(ReportFormat reportFormat, String reportName) throws Exception { - String name = "/tmp/report.csv"; + String name = "/tmp/report." + reportFormat.getExtension(); try(FileOutputStream fileOutputStream = new FileOutputStream(name)) { - QInstance qInstance = QContext.getQInstance(); - qInstance.addReport(definePersonShoesPivotReport(true)); - insertPersonRecords(qInstance); - ReportInput reportInput = new ReportInput(); - reportInput.setReportName(REPORT_NAME); - reportInput.setReportFormat(ReportFormat.CSV); - reportInput.setReportOutputStream(fileOutputStream); + reportInput.setReportName(reportName); + reportInput.setReportDestination(new ReportDestination().withReportFormat(reportFormat).withReportOutputStream(fileOutputStream)); reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); new GenerateReportAction().execute(reportInput); System.out.println("Wrote File: " + name); + return (FileUtils.readFileToString(new File(name), StandardCharsets.UTF_8)); } } @@ -341,27 +360,189 @@ public class GenerateReportActionTest extends BaseTest ** *******************************************************************************/ @Test - void runToXlsx() throws Exception + void runSummaryToXlsx() throws Exception { - String name = "/tmp/report.xlsx"; + ReportFormat format = ReportFormat.XLSX; + String name = "/tmp/report-" + format + ".xlsx"; try(FileOutputStream fileOutputStream = new FileOutputStream(name)) { QInstance qInstance = QContext.getQInstance(); - qInstance.addReport(definePersonShoesPivotReport(true)); + qInstance.addReport(definePersonShoesSummaryReport(true)); insertPersonRecords(qInstance); ReportInput reportInput = new ReportInput(); reportInput.setReportName(REPORT_NAME); - reportInput.setReportFormat(ReportFormat.XLSX); - reportInput.setReportOutputStream(fileOutputStream); + reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(fileOutputStream)); reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); new GenerateReportAction().execute(reportInput); System.out.println("Wrote File: " + name); + + LocalMacDevUtils.openFile(name); } } + /******************************************************************************* + ** Keep some test coverage on the fastexcel library (as long as we keep it around) + *******************************************************************************/ + @Test + void runTableToXlsxFastexcel() throws Exception + { + ReportFormat format = ReportFormat.XLSX; + String name = "/tmp/report-fastexcel.xlsx"; + try(FileOutputStream fileOutputStream = new FileOutputStream(name)) + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addReport(defineTableOnlyReport()); + insertPersonRecords(qInstance); + + ReportInput reportInput = new ReportInput(); + reportInput.setReportName(REPORT_NAME); + reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(fileOutputStream)); + reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); + reportInput.setOverrideExportStreamerSupplier(ExcelFastexcelExportStreamer::new); + new GenerateReportAction().execute(reportInput); + System.out.println("Wrote File: " + name); + + LocalMacDevUtils.openFile(name); + } + } + + + + /******************************************************************************* + ** Imagine if we used the boldHeaderAndFooter styler + *******************************************************************************/ + @Test + void runTableToXlsxWithOverrideStyles() throws Exception + { + ReportFormat format = ReportFormat.XLSX; + String name = "/tmp/report-fastexcel.xlsx"; + try(FileOutputStream fileOutputStream = new FileOutputStream(name)) + { + QInstance qInstance = QContext.getQInstance(); + QReportMetaData reportMetaData = defineTableOnlyReport(); + reportMetaData.getViews().get(0).withTitleFormat("My Title"); + + qInstance.addReport(reportMetaData); + insertPersonRecords(qInstance); + + ReportInput reportInput = new ReportInput(); + reportInput.setReportName(REPORT_NAME); + reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(fileOutputStream)); + reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); + + reportInput.setOverrideExportStreamerSupplier(() -> new ExcelPoiBasedStreamingExportStreamer() + { + @Override + protected PoiExcelStylerInterface getStylerInterface() + { + return new BoldHeaderAndFooterPoiExcelStyler(); + } + }); + + new GenerateReportAction().execute(reportInput); + System.out.println("Wrote File: " + name); + + LocalMacDevUtils.openFile(name); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void runTableToXlsx() throws Exception + { + ReportFormat format = ReportFormat.XLSX; + String name = "/tmp/report-" + format + ".xlsx"; + try(FileOutputStream fileOutputStream = new FileOutputStream(name)) + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addReport(defineTableOnlyReport()); + insertPersonRecords(qInstance); + + ReportInput reportInput = new ReportInput(); + reportInput.setReportName(REPORT_NAME); + reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(fileOutputStream)); + reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); + new GenerateReportAction().execute(reportInput); + System.out.println("Wrote File: " + name); + + LocalMacDevUtils.openFile(name); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void runPivotToXlsx() throws Exception + { + String name = "/tmp/pivot-test.xlsx"; + try(FileOutputStream fileOutputStream = new FileOutputStream(name)) + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addReport(definePivotReport()); + insertPersonRecords(qInstance); + + ReportInput reportInput = new ReportInput(); + reportInput.setReportName(REPORT_NAME); + reportInput.setReportDestination(new ReportDestination().withReportFormat(ReportFormat.XLSX).withReportOutputStream(fileOutputStream)); + reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); + new GenerateReportAction().execute(reportInput); + System.out.println("Wrote File: " + name); + + LocalMacDevUtils.openFile(name, "/Applications/Numbers.app"); + } + + /////////////////////////////////////////////////////////// + // read the file we wrote, and assert about its contents // + /////////////////////////////////////////////////////////// + FileInputStream file = new FileInputStream(name); + XSSFWorkbook workbook = new XSSFWorkbook(file); + + XSSFSheet sheet = workbook.getSheetAt(1); + List pivotTables = sheet.getPivotTables(); + XSSFPivotTable xssfPivotTable = pivotTables.get(0); + List rowLabelColumns = xssfPivotTable.getRowLabelColumns(); + List colLabelColumns = xssfPivotTable.getColLabelColumns(); + Sheet dataSheet = xssfPivotTable.getDataSheet(); + Sheet parentSheet = xssfPivotTable.getParentSheet(); + System.out.println(); + + Map> data = new HashMap<>(); + int i = 0; + for(Row row : sheet) + { + data.put(i, new ArrayList<>()); + + for(Cell cell : row) + { + data.get(i).add(switch(cell.getCellType()) + { + case _NONE -> "<_NONE>"; + case NUMERIC -> cell.getNumericCellValue(); + case STRING -> cell.getStringCellValue(); + case FORMULA -> cell.getCellFormula(); + case BLANK -> ""; + case BOOLEAN -> cell.getBooleanCellValue(); + case ERROR -> cell.getErrorCellValue(); + }); + } + i++; + } + + System.out.println(data); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -369,8 +550,7 @@ public class GenerateReportActionTest extends BaseTest { ReportInput reportInput = new ReportInput(); reportInput.setReportName(REPORT_NAME); - reportInput.setReportFormat(ReportFormat.LIST_OF_MAPS); - reportInput.setReportOutputStream(new ByteArrayOutputStream()); + reportInput.setReportDestination(new ReportDestination().withReportFormat(ReportFormat.LIST_OF_MAPS).withReportOutputStream(new ByteArrayOutputStream())); reportInput.setInputValues(inputValues); new GenerateReportAction().execute(reportInput); } @@ -380,15 +560,15 @@ public class GenerateReportActionTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - private void insertPersonRecords(QInstance qInstance) throws QException + public static void insertPersonRecords(QInstance qInstance) throws QException { TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of( - new PersonQRecord().withLastName("Jonson").withBirthDate(LocalDate.of(1980, Month.JANUARY, 31)).withNoOfShoes(null).withHomeStateId(1).withPrice(null).withCost(new BigDecimal("0.50")), // wrong last initial - new PersonQRecord().withLastName("Jones").withBirthDate(LocalDate.of(1980, Month.JANUARY, 31)).withNoOfShoes(3).withHomeStateId(1).withPrice(new BigDecimal("1.00")).withCost(new BigDecimal("0.50")), // wrong last initial - new PersonQRecord().withLastName("Kelly").withBirthDate(LocalDate.of(1979, Month.DECEMBER, 30)).withNoOfShoes(4).withHomeStateId(1).withPrice(new BigDecimal("1.20")).withCost(new BigDecimal("0.50")), // bad birthdate - new PersonQRecord().withLastName("Keller").withBirthDate(LocalDate.of(1980, Month.JANUARY, 7)).withNoOfShoes(5).withHomeStateId(1).withPrice(new BigDecimal("2.40")).withCost(new BigDecimal("3.50")), - new PersonQRecord().withLastName("Kelkhoff").withBirthDate(LocalDate.of(1980, Month.FEBRUARY, 15)).withNoOfShoes(6).withHomeStateId(1).withPrice(new BigDecimal("3.60")).withCost(new BigDecimal("3.50")), - new PersonQRecord().withLastName("Kelkhoff").withBirthDate(LocalDate.of(1980, Month.MARCH, 20)).withNoOfShoes(7).withHomeStateId(2).withPrice(new BigDecimal("4.80")).withCost(new BigDecimal("3.50")) + new PersonQRecord().withFirstName("Darin").withLastName("Jonson").withBirthDate(LocalDate.of(1980, Month.JANUARY, 31)).withNoOfShoes(null).withHomeStateId(1).withPrice(null).withCost(new BigDecimal("0.50")), // wrong last initial + new PersonQRecord().withFirstName("Darin").withLastName("Jones").withBirthDate(LocalDate.of(1980, Month.JANUARY, 31)).withNoOfShoes(3).withHomeStateId(1).withPrice(new BigDecimal("1.00")).withCost(new BigDecimal("0.50")), // wrong last initial + new PersonQRecord().withFirstName("Darin").withLastName("Kelly").withBirthDate(LocalDate.of(1979, Month.DECEMBER, 30)).withNoOfShoes(4).withHomeStateId(1).withPrice(new BigDecimal("1.20")).withCost(new BigDecimal("0.50")), // bad birthdate + new PersonQRecord().withFirstName("Trevor").withLastName("Keller").withBirthDate(LocalDate.of(1980, Month.JANUARY, 7)).withNoOfShoes(5).withHomeStateId(1).withPrice(new BigDecimal("2.40")).withCost(new BigDecimal("3.50")), + new PersonQRecord().withFirstName("Trevor").withLastName("Kelkhoff").withBirthDate(LocalDate.of(1980, Month.FEBRUARY, 15)).withNoOfShoes(6).withHomeStateId(1).withPrice(new BigDecimal("3.60")).withCost(new BigDecimal("3.50")), + new PersonQRecord().withFirstName("Kelly").withLastName("Kelkhoff").withBirthDate(LocalDate.of(1980, Month.MARCH, 20)).withNoOfShoes(7).withHomeStateId(2).withPrice(new BigDecimal("4.80")).withCost(new BigDecimal("3.50")) )); } @@ -397,7 +577,7 @@ public class GenerateReportActionTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - public static QReportMetaData definePersonShoesPivotReport(boolean includeTotalRow) + public static QReportMetaData definePersonShoesSummaryReport(boolean includeTotalRow) { return new QReportMetaData() .withName(REPORT_NAME) @@ -416,11 +596,11 @@ public class GenerateReportActionTest extends BaseTest )) .withViews(List.of( new QReportView() - .withName("pivot") - .withLabel("pivot") + .withName("summary") + .withLabel("summary") .withDataSourceName("persons") .withType(ReportType.SUMMARY) - .withPivotFields(List.of("lastName")) + .withSummaryFields(List.of("lastName")) .withIncludeTotalRow(includeTotalRow) .withTitleFormat("Number of shoes - people born between %s and %s - pivot on LastName, sort by Quantity, Revenue DESC") .withTitleFields(List.of("${input.startDate}", "${input.endDate}")) @@ -452,39 +632,14 @@ public class GenerateReportActionTest extends BaseTest @Test void testTableOnlyReport() throws QException { - QInstance qInstance = QContext.getQInstance(); - QReportMetaData report = new QReportMetaData() - .withName(REPORT_NAME) - .withDataSources(List.of( - new QReportDataSource() - .withName("persons") - .withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY) - .withQueryFilter(new QQueryFilter() - .withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, List.of("${input.startDate}"))) - ) - )) - .withInputFields(List.of( - new QFieldMetaData("startDate", QFieldType.DATE_TIME) - )) - .withViews(List.of( - new QReportView() - .withName("table1") - .withLabel("table1") - .withDataSourceName("persons") - .withType(ReportType.TABLE) - .withColumns(List.of( - new QReportField().withName("id"), - new QReportField().withName("firstName"), - new QReportField().withName("lastName") - )) - )); - + QInstance qInstance = QContext.getQInstance(); + QReportMetaData report = defineTableOnlyReport(); qInstance.addReport(report); insertPersonRecords(qInstance); runReport(qInstance, Map.of("startDate", LocalDate.of(1980, Month.JANUARY, 1))); - List> list = ListOfMapsExportStreamer.getList("table1"); + List> list = ListOfMapsExportStreamer.getList("Table 1"); Iterator> iterator = list.iterator(); Map row = iterator.next(); assertEquals(5, list.size()); @@ -493,6 +648,88 @@ public class GenerateReportActionTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + private static QReportMetaData defineTableOnlyReport() + { + QReportMetaData report = new QReportMetaData() + .withName(REPORT_NAME) + .withDataSources(List.of( + new QReportDataSource() + .withName("persons") + .withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, List.of("${input.startDate}")))))) + + .withInputFields(List.of( + new QFieldMetaData("startDate", QFieldType.DATE_TIME))) + + .withViews(List.of( + new QReportView() + .withName("table1") + .withLabel("Table 1") + .withDataSourceName("persons") + .withType(ReportType.TABLE) + .withColumns(List.of( + new QReportField().withName("id"), + new QReportField().withName("firstName"), + new QReportField().withName("lastName"))))); + + return report; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QReportMetaData definePivotReport() + { + QReportMetaData report = new QReportMetaData() + .withName(REPORT_NAME) + .withDataSources(List.of( + new QReportDataSource() + .withName("persons") + .withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, List.of("${input.startDate}")))))) + + .withInputFields(List.of( + new QFieldMetaData("startDate", QFieldType.DATE_TIME))) + + .withViews(List.of( + + new QReportView() + .withName("table1") + .withLabel("Table 1") + .withDataSourceName("persons") + .withType(ReportType.TABLE) + .withColumns(List.of( + new QReportField().withName("id"), + new QReportField().withName("firstName"), + new QReportField().withName("lastName"), + new QReportField().withName("homeStateId") + )), + + new QReportView() + .withName("pivotTable1") + .withLabel("My Pivot Table") + + .withType(ReportType.PIVOT) + .withPivotTableSourceViewName("table1") + .withPivotTableDefinition(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("homeStateId")) + // .withRow(new PivotTableGroupBy().withFieldName("lastName")) + // .withColumn(new PivotTableGroupBy().withFieldName("firstName")) + .withValue(new PivotTableValue().withFunction(PivotTableFunction.COUNT).withFieldName("id"))) + )); + + return report; + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -500,6 +737,180 @@ public class GenerateReportActionTest extends BaseTest void testTwoTableViewsOneDataSourceReport() throws QException { QInstance qInstance = QContext.getQInstance(); + defineTwoViewsOneDataSourceReport(qInstance); + + insertPersonRecords(qInstance); + runReport(qInstance, Map.of("startDate", LocalDate.of(1980, Month.JANUARY, 1))); + + List> list = ListOfMapsExportStreamer.getList("Table 1"); + Iterator> iterator = list.iterator(); + Map row = iterator.next(); + assertEquals(5, list.size()); + assertThat(row).containsOnlyKeys("Id", "First Name", "Last Name"); + + list = ListOfMapsExportStreamer.getList("Table 2"); + iterator = list.iterator(); + row = iterator.next(); + assertEquals(5, list.size()); + assertThat(row).containsOnlyKeys("Birth Date"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneTableViewsOneDataSourceJsonReport() throws Exception + { + QInstance qInstance = QContext.getQInstance(); + QReportMetaData report = defineTableOnlyReport(); + qInstance.addReport(report); + + insertPersonRecords(qInstance); + String json = runToString(ReportFormat.JSON, report.getName()); + // System.out.println(json); + + ///////////////////////////////////////////////////////////////////////////////// + // for a one-view report, we should just have an array of the report's records // + ///////////////////////////////////////////////////////////////////////////////// + JSONArray jsonArray = new JSONArray(json); + assertEquals(6, jsonArray.length()); + assertThat(jsonArray.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("id", 1) + .hasFieldOrPropertyWithValue("firstName", "Darin") + .hasFieldOrPropertyWithValue("lastName", "Jonson"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTwoTableViewsOneDataSourceJsonReport() throws Exception + { + QInstance qInstance = QContext.getQInstance(); + QReportMetaData report = defineTwoViewsOneDataSourceReport(qInstance); + + insertPersonRecords(qInstance); + String json = runToString(ReportFormat.JSON, report.getName()); + // System.out.println(json); + + ///////////////////////////////////////////////////////////////////////////////// + // for a multi-view report, we should have an array with the views as elements // + ///////////////////////////////////////////////////////////////////////////////// + JSONArray jsonArray = new JSONArray(json); + assertEquals(2, jsonArray.length()); + + JSONObject firstView = jsonArray.getJSONObject(0); + assertEquals("Table 1", firstView.getString("name")); + JSONArray firstViewData = firstView.getJSONArray("data"); + assertEquals(6, firstViewData.length()); + assertThat(firstViewData.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("id", 1) + .hasFieldOrPropertyWithValue("firstName", "Darin") + .hasFieldOrPropertyWithValue("lastName", "Jonson"); + + JSONObject secondView = jsonArray.getJSONObject(1); + assertEquals("Table 2", secondView.getString("name")); + JSONArray secondViewData = secondView.getJSONArray("data"); + assertEquals(6, secondViewData.length()); + assertThat(secondViewData.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("birthDate", "1980-01-31"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableViewsAndSummaryViewJsonReport() throws Exception + { + QInstance qInstance = QContext.getQInstance(); + QReportMetaData report = defineSimplePersonTableAndSummaryByFirstNameReport(); + qInstance.addReport(report); + + insertPersonRecords(qInstance); + String json = runToString(ReportFormat.JSON, report.getName()); + System.out.println(json); + + ///////////////////////////////////////////////////////////////////////////////// + // for a multi-view report, we should have an array with the views as elements // + ///////////////////////////////////////////////////////////////////////////////// + JSONArray jsonArray = new JSONArray(json); + assertEquals(2, jsonArray.length()); + + JSONObject firstView = jsonArray.getJSONObject(0); + assertEquals("Table 1", firstView.getString("name")); + JSONArray firstViewData = firstView.getJSONArray("data"); + assertEquals(6, firstViewData.length()); + assertThat(firstViewData.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("id", 1) + .hasFieldOrPropertyWithValue("firstName", "Darin") + .hasFieldOrPropertyWithValue("lastName", "Jonson"); + + JSONObject secondView = jsonArray.getJSONObject(1); + assertEquals("Summary", secondView.getString("name")); + JSONArray secondViewData = secondView.getJSONArray("data"); + assertEquals(4, secondViewData.length()); + assertThat(secondViewData.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("firstName", "Darin") + .hasFieldOrPropertyWithValue("personCount", 3); + assertThat(secondViewData.getJSONObject(3).toMap()) + .hasFieldOrPropertyWithValue("firstName", "Totals") + .hasFieldOrPropertyWithValue("personCount", 6); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QReportMetaData defineSimplePersonTableAndSummaryByFirstNameReport() + { + return new QReportMetaData() + .withName(REPORT_NAME) + .withDataSources(List.of( + new QReportDataSource() + .withName("persons") + .withSourceTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilter(new QQueryFilter()) + )) + .withViews(List.of( + new QReportView() + .withName("table1") + .withLabel("Table 1") + .withDataSourceName("persons") + .withType(ReportType.TABLE) + .withColumns(List.of( + new QReportField().withName("id"), + new QReportField().withName("firstName"), + new QReportField().withName("lastName"), + new QReportField().withName("homeStateId") + )), + new QReportView() + .withName("summary") + .withLabel("Summary") + .withDataSourceName("persons") + .withType(ReportType.SUMMARY) + .withSummaryFields(List.of("firstName")) + .withIncludeTotalRow(true) + .withOrderByFields(List.of(new QFilterOrderBy("personCount", false))) + .withColumns(List.of( + new QReportField().withName("personCount").withLabel("Person Count").withFormula("${pivot.count.id}").withDisplayFormat(DisplayFormat.COMMAS) + )) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QReportMetaData defineTwoViewsOneDataSourceReport(QInstance qInstance) + { QReportMetaData report = new QReportMetaData() .withName(REPORT_NAME) .withDataSources(List.of( @@ -516,7 +927,7 @@ public class GenerateReportActionTest extends BaseTest .withViews(List.of( new QReportView() .withName("table1") - .withLabel("table1") + .withLabel("Table 1") .withDataSourceName("persons") .withType(ReportType.TABLE) .withColumns(List.of( @@ -526,7 +937,7 @@ public class GenerateReportActionTest extends BaseTest )), new QReportView() .withName("table2") - .withLabel("table2") + .withLabel("Table 2") .withDataSourceName("persons") .withType(ReportType.TABLE) .withColumns(List.of( @@ -535,21 +946,7 @@ public class GenerateReportActionTest extends BaseTest )); qInstance.addReport(report); - - insertPersonRecords(qInstance); - runReport(qInstance, Map.of("startDate", LocalDate.of(1980, Month.JANUARY, 1))); - - List> list = ListOfMapsExportStreamer.getList("table1"); - Iterator> iterator = list.iterator(); - Map row = iterator.next(); - assertEquals(5, list.size()); - assertThat(row).containsOnlyKeys("Id", "First Name", "Last Name"); - - list = ListOfMapsExportStreamer.getList("table2"); - iterator = list.iterator(); - row = iterator.next(); - assertEquals(5, list.size()); - assertThat(row).containsOnlyKeys("Birth Date"); + return (report); } @@ -566,8 +963,7 @@ public class GenerateReportActionTest extends BaseTest ReportInput reportInput = new ReportInput(); reportInput.setReportName(TestUtils.REPORT_NAME_PERSON_SIMPLE); - reportInput.setReportFormat(ReportFormat.LIST_OF_MAPS); - reportInput.setReportOutputStream(new ByteArrayOutputStream()); + reportInput.setReportDestination(new ReportDestination().withReportFormat(ReportFormat.LIST_OF_MAPS).withReportOutputStream(new ByteArrayOutputStream())); new GenerateReportAction().execute(reportInput); List> list = ListOfMapsExportStreamer.getList("Simple Report"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamerTest.java new file mode 100644 index 00000000..126272bc --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamerTest.java @@ -0,0 +1,54 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for JsonExportStreamer + *******************************************************************************/ +class JsonExportStreamerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + Function runOne = label -> new JsonExportStreamer().getLabelForJson(new QFieldMetaData("test", QFieldType.STRING).withLabel(label)); + assertEquals("sku", runOne.apply("SKU")); + assertEquals("clientName", runOne.apply("Client Name")); + assertEquals("slaStatus", runOne.apply("SLA Status")); + assertEquals("lineItem:sku", runOne.apply("Line Item: SKU")); + assertEquals("parcel:slaStatus", runOne.apply("Parcel: SLA Status")); + assertEquals("order:client", runOne.apply("Order: Client")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java index 4ce0e8ee..da57c703 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceActionTest.java @@ -43,7 +43,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; /******************************************************************************* - ** Unit test for ReplaceAction + ** Unit test for ReplaceAction *******************************************************************************/ class ReplaceActionTest extends BaseTest { @@ -157,6 +157,134 @@ class ReplaceActionTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTwoKeysWithNullsNotMatchingAllowingDelete() throws QException + { + String tableName = TestUtils.TABLE_NAME_TWO_KEYS; + + //////////////////////////////// + // start with these 2 records // + //////////////////////////////// + new InsertAction().execute(new InsertInput(tableName).withRecords(List.of( + new QRecord().withValue("key1", 1).withValue("key2", 2), + new QRecord().withValue("key1", 3) + ))); + + //////////////////////////////////////////////////// + // now do a replace action that just updates them // + //////////////////////////////////////////////////// + List newThings = List.of( + new QRecord().withValue("key1", 1).withValue("key2", 2), + new QRecord().withValue("key1", 3) + ); + + ////////////////////////////// + // replace allowing deletes // + ////////////////////////////// + ReplaceInput replaceInput = new ReplaceInput(); + replaceInput.setTableName(tableName); + replaceInput.setKey(new UniqueKey("key1", "key2")); + replaceInput.setOmitDmlAudit(true); + replaceInput.setRecords(newThings); + replaceInput.setFilter(null); + ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); + + assertEquals(1, replaceOutput.getInsertOutput().getRecords().size()); + assertEquals(1, replaceOutput.getUpdateOutput().getRecords().size()); + assertEquals(1, replaceOutput.getDeleteOutput().getDeletedRecordCount()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTwoKeysWithNullsNotMatchingNotAllowingDelete() throws QException + { + String tableName = TestUtils.TABLE_NAME_TWO_KEYS; + + //////////////////////////////// + // start with these 2 records // + //////////////////////////////// + new InsertAction().execute(new InsertInput(tableName).withRecords(List.of( + new QRecord().withValue("key1", 1).withValue("key2", 2), + new QRecord().withValue("key1", 3) + ))); + + //////////////////////////////////////////////////// + // now do a replace action that just updates them // + //////////////////////////////////////////////////// + List newThings = List.of( + new QRecord().withValue("key1", 1).withValue("key2", 2), + new QRecord().withValue("key1", 3) + ); + + ///////////////////////////////// + // replace disallowing deletes // + ///////////////////////////////// + ReplaceInput replaceInput = new ReplaceInput(); + replaceInput.setTableName(tableName); + replaceInput.setKey(new UniqueKey("key1", "key2")); + replaceInput.setOmitDmlAudit(true); + replaceInput.setRecords(newThings); + replaceInput.setFilter(null); + replaceInput.setPerformDeletes(false); + ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); + + assertEquals(1, replaceOutput.getInsertOutput().getRecords().size()); + assertEquals(1, replaceOutput.getUpdateOutput().getRecords().size()); + assertNull(replaceOutput.getDeleteOutput()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTwoKeysWithNullMatching() throws QException + { + String tableName = TestUtils.TABLE_NAME_TWO_KEYS; + + //////////////////////////////// + // start with these 2 records // + //////////////////////////////// + new InsertAction().execute(new InsertInput(tableName).withRecords(List.of( + new QRecord().withValue("key1", 1).withValue("key2", 2), + new QRecord().withValue("key1", 3) + ))); + + //////////////////////////////////////////////////// + // now do a replace action that just updates them // + //////////////////////////////////////////////////// + List newThings = List.of( + new QRecord().withValue("key1", 1).withValue("key2", 2), + new QRecord().withValue("key1", 3) + ); + + /////////////////////////////////////////////// + // replace treating null key values as equal // + /////////////////////////////////////////////// + ReplaceInput replaceInput = new ReplaceInput(); + replaceInput.setTableName(tableName); + replaceInput.setKey(new UniqueKey("key1", "key2")); + replaceInput.setOmitDmlAudit(true); + replaceInput.setRecords(newThings); + replaceInput.setFilter(null); + replaceInput.setAllowNullKeyValuesToEqual(true); + ReplaceOutput replaceOutput = new ReplaceAction().execute(replaceInput); + + assertEquals(0, replaceOutput.getInsertOutput().getRecords().size()); + assertEquals(2, replaceOutput.getUpdateOutput().getRecords().size()); + assertEquals(0, replaceOutput.getDeleteOutput().getDeletedRecordCount()); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -297,4 +425,4 @@ class ReplaceActionTest extends BaseTest return new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withUniqueKey(Map.of("firstName", firstName, "lastName", lastName))).getValueInteger("noOfShoes"); } -} \ No newline at end of file +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java index 83dd98ef..4249275c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.actions.templates.RenderTemplateInpu import com.kingsrook.qqq.backend.core.model.actions.templates.RenderTemplateOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.templates.TemplateType; +import com.kingsrook.qqq.backend.core.utils.LocalMacDevUtils; import org.junit.jupiter.api.Test; @@ -107,8 +108,8 @@ class ConvertHtmlToPdfActionTest extends BaseTest ///////////////////////////////////////////////////////////////////////// // for local dev on a mac, turn this on to auto-open the generated PDF // ///////////////////////////////////////////////////////////////////////// - // todo not commit - // Runtime.getRuntime().exec(new String[] { "/usr/bin/open", "/tmp/file.pdf" }); + // LocalMacDevUtils.mayOpenFiles = true; + LocalMacDevUtils.openFile("/tmp/file.pdf"); } } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java index 6e52c49d..1015412c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java @@ -216,7 +216,7 @@ class QInstanceHelpContentManagerTest extends BaseTest // now - post-insert customizer should have automatically added help content to the instance // /////////////////////////////////////////////////////////////////////////////////////////////// assertTrue(widget.getHelpContent().containsKey("label")); - assertEquals("i need somebody", widget.getHelpContent().get("label").getContent()); + assertEquals("i need somebody", widget.getHelpContent().get("label").get(0).getContent()); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index ba9a6aa4..e0eec66e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -272,6 +272,25 @@ public class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** Test rules for process step names (must be set; must not be duplicated) + ** + *******************************************************************************/ + @Test + public void test_validateProcessStepNames() + { + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(0).setName(null), + "Missing name for a step at index"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(0).setName(""), + "Missing name for a step at index"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().forEach(s -> s.setName("myStep")), + "Duplicate step name [myStep]", "Duplicate step name [myStep]"); + } + + + /******************************************************************************* ** Test that a process with a step that is a private class fails ** diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehaviorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehaviorTest.java index ca31fdc3..a7afdb22 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehaviorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DateTimeDisplayValueBehaviorTest.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; /******************************************************************************* @@ -101,6 +102,48 @@ class DateTimeDisplayValueBehaviorTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadZoneIdFromOtherField() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + table.withField(new QFieldMetaData("timeZone", QFieldType.STRING)); + table.getField("createDate").withBehavior(new DateTimeDisplayValueBehavior().withZoneIdFromFieldName("timeZone")); + + QRecord record = new QRecord().withValue("createDate", Instant.parse("2024-04-04T19:12:00Z")).withValue("timeZone", "fail"); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null); + assertNull(record.getDisplayValue("createDate")); + + record = new QRecord().withValue("createDate", Instant.parse("2024-04-04T19:12:00Z")).withValue("timeZone", null); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null); + assertNull(record.getDisplayValue("createDate")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNullValue() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + table.withField(new QFieldMetaData("timeZone", QFieldType.STRING)); + table.getField("createDate").withBehavior(new DateTimeDisplayValueBehavior().withZoneIdFromFieldName("timeZone")); + + QRecord record = new QRecord().withValue("createDate", null).withValue("timeZone", "UTC"); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null); + assertNull(record.getDisplayValue("createDate")); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -108,8 +151,8 @@ class DateTimeDisplayValueBehaviorTest extends BaseTest void testValidation() { QInstance qInstance = QContext.getQInstance(); - QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); - QFieldMetaData field = table.getField("createDate"); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + QFieldMetaData field = table.getField("createDate"); table.withField(new QFieldMetaData("timeZone", QFieldType.STRING)); Function, List> testOne = setup -> diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehaviorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehaviorTest.java index ad07b882..39d0af14 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehaviorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehaviorTest.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -156,4 +157,47 @@ class DynamicDefaultValueBehaviorTest extends BaseTest assertNull(record.getValue("firstName")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUserId() + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + table.getField("firstName").withBehavior(DynamicDefaultValueBehavior.USER_ID); + + { + //////////////////////////////// + // set it (if null) on insert // + //////////////////////////////// + QRecord record = new QRecord().withValue("id", 1); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null); + assertEquals(QContext.getQSession().getUser().getIdReference(), record.getValue("firstName")); + } + + { + //////////////////////////////// + // set it (if null) on update // + //////////////////////////////// + QRecord record = new QRecord().withValue("id", 1); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record), null); + assertEquals(QContext.getQSession().getUser().getIdReference(), record.getValue("firstName")); + } + + { + //////////////////////////////////////////////////////////////////// + // only set it if it wasn't previously set (both insert & update) // + //////////////////////////////////////////////////////////////////// + QRecord record = new QRecord().withValue("id", 1).withValue("firstName", "Bob"); + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null); + assertEquals("Bob", record.getValue("firstName")); + + ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record), null); + assertEquals("Bob", record.getValue("firstName")); + } + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportJsonFieldDisplayValueFormatterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportJsonFieldDisplayValueFormatterTest.java new file mode 100644 index 00000000..4f00be6e --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportJsonFieldDisplayValueFormatterTest.java @@ -0,0 +1,166 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for SavedReportJsonFieldDisplayValueFormatter + *******************************************************************************/ +class SavedReportJsonFieldDisplayValueFormatterTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws QException + { + QContext.getQInstance().add(new SavedReportsMetaDataProvider().defineSavedReportTable(TestUtils.MEMORY_BACKEND_NAME, null)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostQuery() throws QException + { + UnsafeFunction customize = savedReport -> + { + QInstance qInstance = QContext.getQInstance(); + QTableMetaData table = qInstance.getTable(SavedReport.TABLE_NAME); + + QRecord record = savedReport.toQRecord(); + + for(String fieldName : List.of("queryFilterJson", "columnsJson", "pivotTableJson")) + { + SavedReportJsonFieldDisplayValueFormatter.getInstance().apply(ValueBehaviorApplier.Action.FORMATTING, List.of(record), qInstance, table, table.getField(fieldName)); + } + + return (record); + }; + + { + QRecord record = customize.apply(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withColumnsJson(JsonUtils.toJson(new ReportColumns())) + .withPivotTableJson(JsonUtils.toJson(new PivotTableDefinition()))); + + assertEquals("0 Filters", record.getDisplayValue("queryFilterJson")); + assertEquals("0 Columns", record.getDisplayValue("columnsJson")); + assertEquals("0 Rows, 0 Columns, and 0 Values", record.getDisplayValue("pivotTableJson")); + } + + { + QRecord record = customize.apply(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withColumnsJson(JsonUtils.toJson(new ReportColumns()))); + + assertEquals("0 Filters", record.getDisplayValue("queryFilterJson")); + assertEquals("0 Columns", record.getDisplayValue("columnsJson")); + assertNull(record.getDisplayValue("pivotTableJson")); + } + + { + QRecord record = customize.apply(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter() + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.IS_NOT_BLANK)))) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn(new ReportColumn().withName("birthDate")))) + .withPivotTableJson(JsonUtils.toJson(new PivotTableDefinition() + .withRow(new PivotTableGroupBy()) + .withValue(new PivotTableValue()) + ))); + + assertEquals("1 Filter", record.getDisplayValue("queryFilterJson")); + assertEquals("1 Column", record.getDisplayValue("columnsJson")); + assertEquals("1 Row, 0 Columns, and 1 Value", record.getDisplayValue("pivotTableJson")); + } + + { + QRecord record = customize.apply(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter() + .withCriteria(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, 1)) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.IS_NOT_BLANK)))) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn(new ReportColumn().withName("__check__").withIsVisible(true)) + .withColumn(new ReportColumn().withName("id")) + .withColumn(new ReportColumn().withName("firstName").withIsVisible(true)) + .withColumn(new ReportColumn().withName("lastName").withIsVisible(false)) + .withColumn(new ReportColumn().withName("birthDate")))) + .withPivotTableJson(JsonUtils.toJson(new PivotTableDefinition() + .withRow(new PivotTableGroupBy()) + .withRow(new PivotTableGroupBy()) + .withColumn(new PivotTableGroupBy()) + .withValue(new PivotTableValue()) + .withValue(new PivotTableValue()) + .withValue(new PivotTableValue()) + ))); + + assertEquals("2 Filters", record.getDisplayValue("queryFilterJson")); + assertEquals("3 Columns", record.getDisplayValue("columnsJson")); + assertEquals("2 Rows, 1 Column, and 3 Values", record.getDisplayValue("pivotTableJson")); + } + + { + QRecord record = customize.apply(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson("blah") + .withColumnsJson("") + .withPivotTableJson("{]")); + + assertEquals("Invalid Filter...", record.getDisplayValue("queryFilterJson")); + assertEquals("Invalid Columns...", record.getDisplayValue("columnsJson")); + assertEquals("Invalid Pivot Table...", record.getDisplayValue("pivotTableJson")); + } + + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizerTest.java new file mode 100644 index 00000000..6bf5d188 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportTableCustomizerTest.java @@ -0,0 +1,228 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedreports; + + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableFunction; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for SavedReportTableCustomizer + *******************************************************************************/ +class SavedReportTableCustomizerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws QException + { + QContext.getQInstance().add(new SavedReportsMetaDataProvider().defineSavedReportTable(TestUtils.MEMORY_BACKEND_NAME, null)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPreInsertAndPreUpdateAreWired() throws QException + { + SavedReport badRecord = new SavedReport() + .withLabel("My Report") + .withTableName("notATable"); + + ///////////////////////////////////////////////////////////////////// + // assertions to apply both to a failed insert and a failed update // + ///////////////////////////////////////////////////////////////////// + Consumer asserter = record -> assertThat(record.getErrors()) + .hasSizeGreaterThanOrEqualTo(2) + .anyMatch(e -> e.getMessage().contains("Unrecognized table name")) + .anyMatch(e -> e.getMessage().contains("must contain at least 1 column")); + + //////////////////////////////////////////////////////////// + // go through insert action, to ensure wired-up correctly // + //////////////////////////////////////////////////////////// + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(badRecord)); + asserter.accept(insertOutput.getRecords().get(0)); + + //////////////////////////////// + // likewise for update action // + //////////////////////////////// + UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(SavedReport.TABLE_NAME).withRecordEntity(badRecord)); + asserter.accept(updateOutput.getRecords().get(0)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testParseFails() + { + QRecord record = new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson("...") + .withColumnsJson("x") + .withPivotTableJson("[") + .toQRecord(); + + new SavedReportTableCustomizer().preValidateRecord(record); + + assertThat(record.getErrors()) + .hasSize(3) + .anyMatch(e -> e.getMessage().contains("Unable to parse queryFilterJson")) + .anyMatch(e -> e.getMessage().contains("Unable to parse columnsJson")) + .anyMatch(e -> e.getMessage().contains("Unable to parse pivotTableJson")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNoColumns() + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // given a reportColumns object, serialize it to json, put it in a saved report record, and run the pre-validator // + // then assert we got error saying there were no columns. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Consumer asserter = reportColumns -> + { + SavedReport savedReport = new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withColumnsJson(JsonUtils.toJson(reportColumns)); + + QRecord record = savedReport.toQRecord(); + new SavedReportTableCustomizer().preValidateRecord(record); + + assertThat(record.getErrors()) + .hasSize(1) + .anyMatch(e -> e.getMessage().contains("must contain at least 1 column")); + }; + + asserter.accept(new ReportColumns()); + asserter.accept(new ReportColumns().withColumns(null)); + asserter.accept(new ReportColumns().withColumns(new ArrayList<>())); + asserter.accept(new ReportColumns().withColumn(new ReportColumn() + .withName("id").withIsVisible(false))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPivotTables() + { + BiConsumer> asserter = (PivotTableDefinition ptd, List expectedAnyMessageToContain) -> + { + SavedReport savedReport = new SavedReport() + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("firstName") + .withColumn("lastName") + .withColumn("birthDate"))) + .withPivotTableJson(JsonUtils.toJson(ptd)); + + QRecord record = savedReport.toQRecord(); + new SavedReportTableCustomizer().preValidateRecord(record); + + assertThat(record.getErrors()).hasSize(expectedAnyMessageToContain.size()); + + for(String expected : expectedAnyMessageToContain) + { + assertThat(record.getErrors()) + .anyMatch(e -> e.getMessage().contains(expected)); + } + }; + + asserter.accept(new PivotTableDefinition(), List.of("must contain at least 1 row")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withRow(new PivotTableGroupBy()), + List.of("Missing field name for at least one pivot table row")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withRow(new PivotTableGroupBy().withFieldName("createDate")), + List.of("row is using field (Create Date) which is not an active column")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withColumn(new PivotTableGroupBy()), + List.of("Missing field name for at least one pivot table column")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withColumn(new PivotTableGroupBy().withFieldName("createDate")), + List.of("column is using field (Create Date) which is not an active column")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withValue(new PivotTableValue().withFunction(PivotTableFunction.SUM)), + List.of("Missing field name for at least one pivot table value")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withValue(new PivotTableValue().withFieldName("createDate").withFunction(PivotTableFunction.SUM)), + List.of("value is using field (Create Date) which is not an active column")); + + asserter.accept(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("id")) + .withValue(new PivotTableValue().withFieldName("firstName")), + List.of("Missing function for at least one pivot table value")); + } + + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java index 7e926fe1..09203b82 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java @@ -45,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobsMetaDataProvider; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -97,7 +98,7 @@ class ScheduledJobTableCustomizerTest extends BaseTest @AfterEach void afterEach() { - QuartzTestUtils.afterEach(); + SchedulerTestUtils.afterEach(); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountActionTest.java index 0d0336b9..b1ff66ab 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationCountActionTest.java @@ -89,7 +89,7 @@ class EnumerationCountActionTest extends BaseTest QInstance instance = QContext.getQInstance(); instance.addBackend(new QBackendMetaData() .withName("enum") - .withBackendType("enum") + .withBackendType(EnumerationBackendModule.class) ); instance.addTable(new QTableMetaData() diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java index 909c9f7c..cf44a7b6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/enumeration/EnumerationQueryActionTest.java @@ -167,7 +167,7 @@ class EnumerationQueryActionTest extends BaseTest QInstance instance = QContext.getQInstance(); instance.addBackend(new QBackendMetaData() .withName("enum") - .withBackendType("enum") + .withBackendType(EnumerationBackendModule.class) ); instance.addTable(new QTableMetaData() diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcessTest.java index de193fc6..d84e0a97 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcessTest.java @@ -50,7 +50,7 @@ class BasicRunReportProcessTest extends BaseTest void testRunReport() throws QException { QInstance instance = TestUtils.defineInstance(); - QReportMetaData report = GenerateReportActionTest.definePersonShoesPivotReport(true); + QReportMetaData report = GenerateReportActionTest.definePersonShoesSummaryReport(true); QProcessMetaData runReportProcess = BasicRunReportProcess.defineProcessMetaData(); instance.addReport(report); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportProcessTest.java new file mode 100644 index 00000000..3e6efae7 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportProcessTest.java @@ -0,0 +1,406 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableFunction; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryStorageAction; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.LocalMacDevUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.apache.commons.io.IOUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for RenderSavedReportExecuteStep + *******************************************************************************/ +class RenderSavedReportProcessTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + GenerateReportActionTest.insertPersonRecords(QContext.getQInstance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + @Disabled + void testForDevPrintAPivotDefinitionAsJson() + { + System.out.println(JsonUtils.toPrettyJson(new PivotTableDefinition() + .withRow(new PivotTableGroupBy() + .withFieldName("homeStateId")) + .withRow(new PivotTableGroupBy() + .withFieldName("firstName")) + .withValue(new PivotTableValue() + .withFieldName("id") + .withFunction(PivotTableFunction.COUNT)) + .withValue(new PivotTableValue() + .withFieldName("cost") + .withFunction(PivotTableFunction.SUM)) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableOnlyReport() throws Exception + { + String label = "Test Report"; + + ////////////////////////////////////////////////////////////////////////////////////////// + // do columns json as a string, rather than a toJson'ed ReportColumns object, // + // to help verify that we don't choke on un-recognized properties (e.g., as QFMD sends) // + ////////////////////////////////////////////////////////////////////////////////////////// + String columnsJson = """ + {"columns":[ + {"name": "k"}, + {"name": "id"}, + {"name": "firstName", "isVisible": true}, + {"name": "lastName", "pinned": "left"}, + {"name": "createDate", "isVisible": false} + ]}"""; + + QRecord savedReport = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(new SavedReport() + .withLabel(label) + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withColumnsJson(columnsJson) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + )).getRecords().get(0); + + RunProcessOutput runProcessOutput = runRenderReportProcess(savedReport, ReportFormatPossibleValueEnum.CSV); + + String downloadFileName = runProcessOutput.getValueString("downloadFileName"); + assertThat(downloadFileName) + .startsWith(label + " - ") + .matches(".*\\d\\d\\d\\d-\\d\\d-\\d\\d-\\d\\d\\d\\d.*") + .endsWith(".csv"); + + InputStream inputStream = getInputStream(runProcessOutput); + List lines = IOUtils.readLines(inputStream, StandardCharsets.UTF_8); + + assertEquals(""" + "Id","First Name","Last Name" + """.trim(), lines.get(0)); + assertEquals(""" + "1","Darin","Jonson" + """.trim(), lines.get(1)); + + writeTmpFileAndOpen(inputStream, ".csv"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static InputStream getInputStream(RunProcessOutput runProcessOutput) throws QException + { + String storageTableName = runProcessOutput.getValueString("storageTableName"); + String storageReference = runProcessOutput.getValueString("storageReference"); + InputStream inputStream = new MemoryStorageAction().getInputStream(new StorageInput(storageTableName).withReference(storageReference)); + return inputStream; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void writeTmpFileAndOpen(InputStream inputStream, String suffix) throws IOException + { + // LocalMacDevUtils.mayOpenFiles = true; + if(LocalMacDevUtils.mayOpenFiles) + { + inputStream.reset(); + + File tmpFile = File.createTempFile(getClass().getName(), suffix, new File("/tmp/")); + FileOutputStream fileOutputStream = new FileOutputStream(tmpFile); + inputStream.transferTo(fileOutputStream); + fileOutputStream.close(); + + LocalMacDevUtils.openFile(tmpFile.getAbsolutePath()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QRecord insertBasicSavedPivotReport(String label) throws QException + { + return new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(new SavedReport() + .withLabel(label) + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("firstName") + .withColumn("lastName") + .withColumn("cost") + .withColumn("birthDate") + .withColumn("homeStateId"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withPivotTableJson(JsonUtils.toJson(new PivotTableDefinition() + .withRow(new PivotTableGroupBy() + .withFieldName("homeStateId")) + .withRow(new PivotTableGroupBy() + .withFieldName("firstName")) + .withValue(new PivotTableValue() + .withFieldName("id") + .withFunction(PivotTableFunction.COUNT)) + .withValue(new PivotTableValue() + .withFieldName("cost") + .withFunction(PivotTableFunction.SUM)) + .withValue(new PivotTableValue() + .withFieldName("birthDate") + .withFunction(PivotTableFunction.MIN)) + .withValue(new PivotTableValue() + .withFieldName("birthDate") + .withFunction(PivotTableFunction.MAX)) + )) + )).getRecords().get(0); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPivotXlsx() throws Exception + { + String label = "Test Pivot Report"; + QRecord savedReport = insertBasicSavedPivotReport(label); + RunProcessOutput runProcessOutput = runRenderReportProcess(savedReport, ReportFormatPossibleValueEnum.XLSX); + + InputStream inputStream = getInputStream(runProcessOutput); + writeTmpFileAndOpen(inputStream, ".xlsx"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPivotJson() throws Exception + { + String label = "Test Pivot Report JSON"; + QRecord savedReport = insertBasicSavedPivotReport(label); + RunProcessOutput runProcessOutput = runRenderReportProcess(savedReport, ReportFormatPossibleValueEnum.JSON); + + InputStream inputStream = getInputStream(runProcessOutput); + String json = StringUtils.join("\n", IOUtils.readLines(inputStream, StandardCharsets.UTF_8)); + printReport(json); + + JSONArray jsonArray = new JSONArray(json); + assertEquals(2, jsonArray.length()); + + JSONObject firstView = jsonArray.getJSONObject(0); + assertEquals(label, firstView.getString("name")); + JSONArray firstViewData = firstView.getJSONArray("data"); + assertEquals(6, firstViewData.length()); + assertThat(firstViewData.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("id", 1) + .hasFieldOrPropertyWithValue("firstName", "Darin"); + + JSONObject pivotView = jsonArray.getJSONObject(1); + assertEquals("Pivot Table", pivotView.getString("name")); + JSONArray pivotViewData = pivotView.getJSONArray("data"); + assertEquals(4, pivotViewData.length()); + assertThat(pivotViewData.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("homeState", "IL") + .hasFieldOrPropertyWithValue("firstName", "Darin") + .hasFieldOrPropertyWithValue("countOfId", 3) + .hasFieldOrPropertyWithValue("sumOfCost", new BigDecimal("1.50")); + assertThat(pivotViewData.getJSONObject(3).toMap()) + .hasFieldOrPropertyWithValue("homeState", "Totals") + .hasFieldOrPropertyWithValue("countOfId", 6) + .hasFieldOrPropertyWithValue("sumOfCost", new BigDecimal("12.00")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void printReport(String report) + { + // System.out.println(report); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPivotCSV() throws Exception + { + String label = "Test Pivot Report CSV"; + QRecord savedReport = insertBasicSavedPivotReport(label); + RunProcessOutput runProcessOutput = runRenderReportProcess(savedReport, ReportFormatPossibleValueEnum.CSV); + + InputStream inputStream = getInputStream(runProcessOutput); + List csv = IOUtils.readLines(inputStream, StandardCharsets.UTF_8); + System.out.println(csv); + + assertEquals(""" + "Home State","First Name","Count Of Id","Sum Of Cost","Min Of Birth Date","Max Of Birth Date\"""", csv.get(0)); + + assertEquals(""" + "Totals","","6","12.00","1979-12-30","1980-03-20\"""", csv.get(4)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QRecord insertSavedPivotReportWithAllFunctions(String label) throws QException + { + PivotTableDefinition pivotTableDefinition = new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("firstName")); + + for(PivotTableFunction function : PivotTableFunction.values()) + { + pivotTableDefinition.withValue(new PivotTableValue().withFieldName("cost").withFunction(function)); + } + + return new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(new SavedReport() + .withLabel(label) + .withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("firstName") + .withColumn("cost"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withPivotTableJson(JsonUtils.toJson(pivotTableDefinition)) + )).getRecords().get(0); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPivotXlsxAllFunctions() throws Exception + { + String label = "Test Pivot Report"; + QRecord savedReport = insertSavedPivotReportWithAllFunctions(label); + RunProcessOutput runProcessOutput = runRenderReportProcess(savedReport, ReportFormatPossibleValueEnum.XLSX); + + String serverFilePath = runProcessOutput.getValueString("serverFilePath"); + System.out.println(serverFilePath); + + InputStream inputStream = getInputStream(runProcessOutput); + writeTmpFileAndOpen(inputStream, ".xlsx"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPivotCSVAllFunctions() throws Exception + { + String label = "Test Pivot Report CSV"; + QRecord savedReport = insertSavedPivotReportWithAllFunctions(label); + RunProcessOutput runProcessOutput = runRenderReportProcess(savedReport, ReportFormatPossibleValueEnum.CSV); + + InputStream inputStream = getInputStream(runProcessOutput); + List csv = IOUtils.readLines(inputStream, StandardCharsets.UTF_8); + System.out.println(csv); + + assertEquals(""" + "First Name","Average Of Cost","Count Of Cost","Count_nums Of Cost","Max Of Cost","Min Of Cost","Product Of Cost","Std_dev Of Cost","Std_devp Of Cost","Sum Of Cost","Var Of Cost","Varp Of Cost\"""", csv.get(0)); + + assertEquals(""" + "Totals","2.0","6","6","3.50","0.50","5.359375000000","1.6432","1.5000","12.00","2.7000","2.2500\"""", csv.get(4)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static RunProcessOutput runRenderReportProcess(QRecord savedReport, ReportFormatPossibleValueEnum reportFormat) throws QException + { + RunProcessInput input = new RunProcessInput(); + input.setProcessName(RenderSavedReportMetaDataProducer.NAME); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.setCallback(QProcessCallbackFactory.forRecord(savedReport)); + input.addValue("reportFormat", reportFormat.getPossibleValueId()); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(input); + return runProcessOutput; + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java index 4b925a5f..b33249e1 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java @@ -54,28 +54,7 @@ class QScheduleManagerTest extends BaseTest void afterEach() { QLogger.deactivateCollectingLoggerForClass(QuartzScheduler.class); - - try - { - QScheduleManager.getInstance().unInit(); - } - catch(IllegalStateException ise) - { - ///////////////////////////////////////////////////////////////// - // ok, might just mean that this test didn't init the instance // - ///////////////////////////////////////////////////////////////// - } - - try - { - QuartzScheduler.getInstance().unInit(); - } - catch(IllegalStateException ise) - { - ///////////////////////////////////////////////////////////////// - // ok, might just mean that this test didn't init the instance // - ///////////////////////////////////////////////////////////////// - } + SchedulerTestUtils.afterEach(); } @@ -178,4 +157,4 @@ class QScheduleManagerTest extends BaseTest .anyMatch(l -> l.getMessage().matches(".*Scheduled new job.*TABLE_AUTOMATIONS.scheduledJob:4.*")); } -} \ No newline at end of file +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java index 3a46ed37..439eb47f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaDa import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobParameter; import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; import com.kingsrook.qqq.backend.core.utils.TestUtils; @@ -86,6 +87,38 @@ public class SchedulerTestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static void afterEach() + { + try + { + QScheduleManager.getInstance().stop(); + QScheduleManager.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + + try + { + QuartzScheduler.getInstance().stop(); + QuartzScheduler.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcessTest.java index 91f5de83..e8153867 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcessTest.java @@ -58,28 +58,7 @@ class RescheduleAllJobsProcessTest extends BaseTest void afterEach() { QLogger.deactivateCollectingLoggerForClass(QuartzScheduler.class); - - try - { - QScheduleManager.getInstance().unInit(); - } - catch(IllegalStateException ise) - { - ///////////////////////////////////////////////////////////////// - // ok, might just mean that this test didn't init the instance // - ///////////////////////////////////////////////////////////////// - } - - try - { - QuartzScheduler.getInstance().unInit(); - } - catch(IllegalStateException ise) - { - ///////////////////////////////////////////////////////////////// - // ok, might just mean that this test didn't init the instance // - ///////////////////////////////////////////////////////////////// - } + SchedulerTestUtils.afterEach(); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcessTest.java new file mode 100644 index 00000000..f9db7c70 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcessTest.java @@ -0,0 +1,157 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobsMetaDataProvider; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableSQSQueueRunner; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.quartz.SchedulerException; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for ScheduleAllNewJobsProcess + *******************************************************************************/ +class ScheduleAllNewJobsProcessTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + SchedulerTestUtils.afterEach(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException, SchedulerException + { + try + { + QCollectingLogger quartzSchedulerLog = QLogger.activateCollectingLoggerForClass(QuartzScheduler.class); + + QInstance qInstance = QContext.getQInstance(); + new ScheduledJobsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, ScheduleAllNewJobsProcess.class.getPackageName()); + QuartzTestUtils.setupInstanceForQuartzTests(); + + /////////////////////////////////////////////////////////////////////////////////// + // clear out the customizers that would normally schedule jobs as we insert them // + /////////////////////////////////////////////////////////////////////////////////// + qInstance.getTable(ScheduledJob.TABLE_NAME).withCustomizers(Collections.emptyMap()); + + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> QContext.getQSession()); + qScheduleManager.start(); + + QuartzScheduler quartzScheduler = QuartzScheduler.getInstance(); + List wrappers = quartzScheduler.queryQuartz(); + + ////////////////////////////////////////////// + // make sure nothing is scheduled initially // + ////////////////////////////////////////////// + assertTrue(wrappers.isEmpty()); + + //////////////////////////////////////////////////////////////////////////// + // insert a scheduled job - run schedule-new, make sure it gets scheduled // + //////////////////////////////////////////////////////////////////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(SchedulerTestUtils + .newScheduledJob(ScheduledJobType.PROCESS, Map.of("processName", TestUtils.PROCESS_NAME_GREET_PEOPLE)) + .withLabel("Test job 1") + .withId(null) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME))); + + RunProcessInput input = new RunProcessInput(); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.setProcessName(ScheduleAllNewJobsProcess.class.getSimpleName()); + new RunProcessAction().execute(input); + + /////////////////////////////////////////////////////////////// + // make sure our scheduledJob here got scheduled with quartz // + /////////////////////////////////////////////////////////////// + wrappers = quartzScheduler.queryQuartz(); + assertEquals(1, wrappers.size()); + assertTrue(wrappers.stream().anyMatch(w -> w.jobDetail().getKey().getName().equals("scheduledJob:1"))); + + /////////////// + // repeat it // + /////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(SchedulerTestUtils + .newScheduledJob(ScheduledJobType.PROCESS, Map.of("processName", TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE)) + .withLabel("Test job 2") + .withId(null) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME))); + + input = new RunProcessInput(); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.setProcessName(ScheduleAllNewJobsProcess.class.getSimpleName()); + new RunProcessAction().execute(input); + + wrappers = quartzScheduler.queryQuartz(); + assertEquals(2, wrappers.size()); + assertTrue(wrappers.stream().anyMatch(w -> w.jobDetail().getKey().getName().equals("scheduledJob:2"))); + + ///////////////////////////////////////////////////////////////////////////////////// + // make sure quartzScheduler never logged about deleting or re-scheduling anything // + ///////////////////////////////////////////////////////////////////////////////////// + assertThat(quartzSchedulerLog.getCollectedMessages()) + .noneMatch(m -> m.getMessage().toLowerCase().contains("delete")) + .noneMatch(m -> m.getMessage().toLowerCase().contains("re-schedule")); + } + finally + { + QLogger.deactivateCollectingLoggerForClass(SchedulableSQSQueueRunner.class); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcessTest.java index 5b69d126..f5774bad 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcessTest.java @@ -28,7 +28,6 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -40,6 +39,7 @@ import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.quartz.SchedulerException; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -52,35 +52,23 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class UnscheduleAllJobsProcessTest extends BaseTest { + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() + { + SchedulerTestUtils.afterEach(); + } + + /******************************************************************************* ** *******************************************************************************/ @AfterEach void afterEach() { - QLogger.deactivateCollectingLoggerForClass(QuartzScheduler.class); - - try - { - QScheduleManager.getInstance().unInit(); - } - catch(IllegalStateException ise) - { - ///////////////////////////////////////////////////////////////// - // ok, might just mean that this test didn't init the instance // - ///////////////////////////////////////////////////////////////// - } - - try - { - QuartzScheduler.getInstance().unInit(); - } - catch(IllegalStateException ise) - { - ///////////////////////////////////////////////////////////////// - // ok, might just mean that this test didn't init the instance // - ///////////////////////////////////////////////////////////////// - } + SchedulerTestUtils.afterEach(); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java index 46f5c8cc..6d971b1a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java @@ -62,7 +62,7 @@ class QuartzSchedulerTest extends BaseTest @AfterEach void afterEach() { - QuartzTestUtils.afterEach(); + SchedulerTestUtils.afterEach(); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java index a82d163c..7e36e6b6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java @@ -27,7 +27,6 @@ import java.util.Properties; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData; -import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; import org.quartz.SchedulerException; @@ -104,34 +103,4 @@ public class QuartzTestUtils return QuartzScheduler.getInstance().queryQuartz(); } - - - /******************************************************************************* - ** - *******************************************************************************/ - public static void afterEach() - { - try - { - QScheduleManager.getInstance().stop(); - QScheduleManager.getInstance().unInit(); - } - catch(IllegalStateException ise) - { - ///////////////////////////////////////////////////////////////// - // ok, might just mean that this test didn't init the instance // - ///////////////////////////////////////////////////////////////// - } - - try - { - QuartzScheduler.getInstance().unInit(); - } - catch(IllegalStateException ise) - { - ///////////////////////////////////////////////////////////////// - // ok, might just mean that this test didn't init the instance // - ///////////////////////////////////////////////////////////////// - } - } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java index f2bf6067..4f53a495 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; @@ -92,7 +93,7 @@ class QuartzJobsProcessTest extends BaseTest @AfterEach void afterEach() { - QuartzTestUtils.afterEach(); + SchedulerTestUtils.afterEach(); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java index 827cc57e..d7622e68 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java @@ -53,7 +53,7 @@ class SimpleSchedulerTest extends BaseTest @AfterEach void afterEach() { - QScheduleManager.getInstance().unInit(); + SchedulerTestUtils.afterEach(); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/testutils/PersonQRecord.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/testutils/PersonQRecord.java index 7f7bdf6f..d5620731 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/testutils/PersonQRecord.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/testutils/PersonQRecord.java @@ -40,6 +40,14 @@ public class PersonQRecord extends QRecord + public PersonQRecord withFirstName(String firstName) + { + setValue("firstName", firstName); + return (this); + } + + + public PersonQRecord withBirthDate(LocalDate birthDate) { setValue("birthDate", birthDate); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ObjectUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ObjectUtilsTest.java index c08486b5..08a308e6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ObjectUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ObjectUtilsTest.java @@ -25,7 +25,9 @@ package com.kingsrook.qqq.backend.core.utils; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* @@ -79,4 +81,20 @@ class ObjectUtilsTest assertEquals("else", ObjectUtils.tryAndRequireNonNullElse(() -> null, "else")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testIfCan() + { + Object nullObject = null; + assertTrue(ObjectUtils.ifCan(() -> true)); + assertTrue(ObjectUtils.ifCan(() -> "a".equals("a"))); + assertFalse(ObjectUtils.ifCan(() -> 1 == 2)); + assertFalse(ObjectUtils.ifCan(() -> nullObject.equals("a"))); + assertFalse(ObjectUtils.ifCan(() -> null)); + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 0086c351..000afa15 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -138,6 +138,7 @@ public class TestUtils public static final String APP_NAME_PEOPLE = "peopleApp"; public static final String APP_NAME_MISCELLANEOUS = "miscellaneous"; + public static final String TABLE_NAME_TWO_KEYS = "twoKeys"; public static final String TABLE_NAME_PERSON = "person"; public static final String TABLE_NAME_SHAPE = "shape"; public static final String TABLE_NAME_SHAPE_CACHE = "shapeCache"; @@ -196,6 +197,7 @@ public class TestUtils qInstance.addBackend(defineMemoryBackend()); qInstance.addTable(defineTablePerson()); + qInstance.addTable(defineTableTwoKeys()); qInstance.addTable(definePersonFileTable()); qInstance.addTable(definePersonMemoryTable()); qInstance.addTable(definePersonMemoryCacheTable()); @@ -545,6 +547,24 @@ public class TestUtils + /******************************************************************************* + ** Define the 'two key' table used in standard tests. + *******************************************************************************/ + public static QTableMetaData defineTableTwoKeys() + { + return new QTableMetaData() + .withName(TABLE_NAME_TWO_KEYS) + .withLabel("Two Keys") + .withBackendName(MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withUniqueKey(new UniqueKey("key1", "key2")) + .withField(new QFieldMetaData("key1", QFieldType.INTEGER)) + .withField(new QFieldMetaData("key2", QFieldType.INTEGER)); + } + + + /******************************************************************************* ** Define the 'person' table used in standard tests. *******************************************************************************/ @@ -791,6 +811,26 @@ public class TestUtils + /******************************************************************************* + ** Define a table with unique key where one is nullable + *******************************************************************************/ + public static QTableMetaData defineTwoKeyTable() + { + return (new QTableMetaData() + .withName(TABLE_NAME_BASEPULL) + .withLabel("Basepull Test") + .withPrimaryKeyField("id") + .withBackendName(MEMORY_BACKEND_NAME) + .withFields(TestUtils.defineTablePerson().getFields())) + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date").withIsEditable(false)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date").withIsEditable(false)) + .withField(new QFieldMetaData(BASEPULL_KEY_FIELD_NAME, QFieldType.STRING).withBackendName("process_name").withIsRequired(true)) + .withField(new QFieldMetaData(BASEPULL_LAST_RUN_TIME_FIELD_NAME, QFieldType.DATE_TIME).withBackendName("last_run_time").withIsRequired(true)); + } + + + /******************************************************************************* ** Define a basepullTable *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java index 86b09257..4dff56c6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java @@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.utils.aggregates; import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.Month; import com.kingsrook.qqq.backend.core.BaseTest; import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; @@ -78,6 +81,12 @@ class AggregatesTest extends BaseTest assertEquals(15, aggregates.getMax()); assertEquals(30, aggregates.getSum()); assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10"), Offset.offset(BigDecimal.ZERO)); + + assertEquals(new BigDecimal("750"), aggregates.getProduct()); + assertEquals(new BigDecimal("25.0000"), aggregates.getVariance()); + assertEquals(new BigDecimal("5.0000"), aggregates.getStandardDeviation()); + assertThat(aggregates.getVarP()).isCloseTo(new BigDecimal("16.6667"), Offset.offset(new BigDecimal(".0001"))); + assertThat(aggregates.getStdDevP()).isCloseTo(new BigDecimal("4.0824"), Offset.offset(new BigDecimal(".0001"))); } @@ -89,6 +98,7 @@ class AggregatesTest extends BaseTest void testBigDecimal() { BigDecimalAggregates aggregates = new BigDecimalAggregates(); + aggregates.add(null); assertEquals(0, aggregates.getCount()); assertNull(aggregates.getMin()); @@ -114,13 +124,117 @@ class AggregatesTest extends BaseTest BigDecimal bd148 = new BigDecimal("14.8"); aggregates.add(bd148); - - aggregates.add(null); assertEquals(3, aggregates.getCount()); assertEquals(bd51, aggregates.getMin()); assertEquals(bd148, aggregates.getMax()); assertEquals(new BigDecimal("30.0"), aggregates.getSum()); assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10.0"), Offset.offset(BigDecimal.ZERO)); + + assertEquals(new BigDecimal("762.348"), aggregates.getProduct()); + assertEquals(new BigDecimal("23.5300"), aggregates.getVariance()); + assertEquals(new BigDecimal("4.8508"), aggregates.getStandardDeviation()); + assertThat(aggregates.getVarP()).isCloseTo(new BigDecimal("15.6867"), Offset.offset(new BigDecimal(".0001"))); + assertThat(aggregates.getStdDevP()).isCloseTo(new BigDecimal("3.9606"), Offset.offset(new BigDecimal(".0001"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testInstant() + { + InstantAggregates aggregates = new InstantAggregates(); + + assertEquals(0, aggregates.getCount()); + assertNull(aggregates.getMin()); + assertNull(aggregates.getMax()); + assertNull(aggregates.getSum()); + assertNull(aggregates.getAverage()); + + Instant i1970 = Instant.parse("1970-01-01T00:00:00Z"); + aggregates.add(i1970); + assertEquals(1, aggregates.getCount()); + assertEquals(i1970, aggregates.getMin()); + assertEquals(i1970, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(i1970, aggregates.getAverage()); + + Instant i1980 = Instant.parse("1980-01-01T00:00:00Z"); + aggregates.add(i1980); + assertEquals(2, aggregates.getCount()); + assertEquals(i1970, aggregates.getMin()); + assertEquals(i1980, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(Instant.parse("1975-01-01T00:00:00Z"), aggregates.getAverage()); + + Instant i1990 = Instant.parse("1990-01-01T00:00:00Z"); + aggregates.add(i1990); + assertEquals(3, aggregates.getCount()); + assertEquals(i1970, aggregates.getMin()); + assertEquals(i1990, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(Instant.parse("1980-01-01T08:00:00Z"), aggregates.getAverage()); // a leap day throws this off by 8 hours :) + + ///////////////////////////////////////////////////////////////////// + // assert we gracefully return null for these ops we don't support // + ///////////////////////////////////////////////////////////////////// + assertNull(aggregates.getProduct()); + assertNull(aggregates.getVariance()); + assertNull(aggregates.getStandardDeviation()); + assertNull(aggregates.getVarP()); + assertNull(aggregates.getStdDevP()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLocalDate() + { + LocalDateAggregates aggregates = new LocalDateAggregates(); + + assertEquals(0, aggregates.getCount()); + assertNull(aggregates.getMin()); + assertNull(aggregates.getMax()); + assertNull(aggregates.getSum()); + assertNull(aggregates.getAverage()); + + LocalDate ld1970 = LocalDate.of(1970, Month.JANUARY, 1); + aggregates.add(ld1970); + assertEquals(1, aggregates.getCount()); + assertEquals(ld1970, aggregates.getMin()); + assertEquals(ld1970, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(ld1970, aggregates.getAverage()); + + LocalDate ld1980 = LocalDate.of(1980, Month.JANUARY, 1); + aggregates.add(ld1980); + assertEquals(2, aggregates.getCount()); + assertEquals(ld1970, aggregates.getMin()); + assertEquals(ld1980, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(LocalDate.of(1975, Month.JANUARY, 1), aggregates.getAverage()); + + LocalDate ld1990 = LocalDate.of(1990, Month.JANUARY, 1); + aggregates.add(ld1990); + assertEquals(3, aggregates.getCount()); + assertEquals(ld1970, aggregates.getMin()); + assertEquals(ld1990, aggregates.getMax()); + assertNull(aggregates.getSum()); + assertEquals(ld1980, aggregates.getAverage()); + + ///////////////////////////////////////////////////////////////////// + // assert we gracefully return null for these ops we don't support // + ///////////////////////////////////////////////////////////////////// + assertNull(aggregates.getProduct()); + assertNull(aggregates.getVariance()); + assertNull(aggregates.getStandardDeviation()); + assertNull(aggregates.getVarP()); + assertNull(aggregates.getStdDevP()); } } \ No newline at end of file diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/APIBackendModule.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/APIBackendModule.java index 486d2db6..9214829c 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/APIBackendModule.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/APIBackendModule.java @@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.module.api.actions.APICountAction; import com.kingsrook.qqq.backend.module.api.actions.APIDeleteAction; @@ -44,6 +45,11 @@ import com.kingsrook.qqq.backend.module.api.actions.APIUpdateAction; *******************************************************************************/ public class APIBackendModule implements QBackendModuleInterface { + static + { + QBackendModuleDispatcher.registerBackendModule(new APIBackendModule()); + } + /******************************************************************************* ** Method where a backend module must be able to provide its type (name). *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java index 820915ff..7888080f 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java @@ -26,11 +26,13 @@ import java.io.File; import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; @@ -39,6 +41,7 @@ import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemCount import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemDeleteAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemInsertAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemQueryAction; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemStorageAction; import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemUpdateAction; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; @@ -53,6 +56,10 @@ public class FilesystemBackendModule implements QBackendModuleInterface, Filesys public static final String BACKEND_TYPE = "filesystem"; + static + { + QBackendModuleDispatcher.registerBackendModule(new FilesystemBackendModule()); + } /******************************************************************************* ** For filesystem backends, get the module-specific action base-class, that helps @@ -152,4 +159,14 @@ public class FilesystemBackendModule implements QBackendModuleInterface, Filesys return (new FilesystemDeleteAction()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QStorageInterface getStorageInterface() + { + return (new FilesystemStorageAction()); + } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageAction.java new file mode 100644 index 00000000..24d9fa47 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageAction.java @@ -0,0 +1,103 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.jetbrains.annotations.NotNull; + + +/******************************************************************************* + ** (mass, streamed) storage action for filesystem module + *******************************************************************************/ +public class FilesystemStorageAction extends AbstractFilesystemAction implements QStorageInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public OutputStream createOutputStream(StorageInput storageInput) throws QException + { + try + { + String fullPath = getFullPath(storageInput); + File file = new File(fullPath); + if(!file.getParentFile().exists()) + { + if(!file.getParentFile().mkdirs()) + { + throw (new QException("Could not make directory required for storing file: " + fullPath)); + } + } + + return (new FileOutputStream(fullPath)); + } + catch(IOException e) + { + throw (new QException("IOException creating output stream for file", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @NotNull + private String getFullPath(StorageInput storageInput) + { + QTableMetaData table = storageInput.getTable(); + QBackendMetaData backend = storageInput.getBackend(); + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + storageInput.getReference()); + return fullPath; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InputStream getInputStream(StorageInput storageInput) throws QException + { + try + { + return (new FileInputStream(getFullPath(storageInput))); + } + catch(IOException e) + { + throw (new QException("IOException getting input stream for file", e)); + } + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java index d613dced..8a9a6272 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java @@ -25,10 +25,12 @@ package com.kingsrook.qqq.backend.module.filesystem.s3; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; @@ -36,6 +38,7 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction; +import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3StorageAction; import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3UpdateAction; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; @@ -48,6 +51,10 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke { public static final String BACKEND_TYPE = "s3"; + static + { + QBackendModuleDispatcher.registerBackendModule(new S3BackendModule()); + } /******************************************************************************* ** For filesystem backends, get the module-specific action base-class, that helps @@ -136,4 +143,15 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke return (new S3DeleteAction()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QStorageInterface getStorageInterface() + { + return new S3StorageAction(); + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index b7b8f999..91383c4b 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -113,7 +113,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction. + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3UploadOutputStream; + + +/******************************************************************************* + ** (mass, streamed) storage action for s3 module + *******************************************************************************/ +public class S3StorageAction extends AbstractS3Action implements QStorageInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public OutputStream createOutputStream(StorageInput storageInput) throws QException + { + try + { + S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend(); + preAction(backend); + + AmazonS3 amazonS3 = getS3Utils().getAmazonS3(); + String fullPath = getFullPath(storageInput); + S3UploadOutputStream s3UploadOutputStream = new S3UploadOutputStream(amazonS3, backend.getBucketName(), fullPath); + return (s3UploadOutputStream); + } + catch(Exception e) + { + throw (new QException("Exception creating s3 output stream for file", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getFullPath(StorageInput storageInput) + { + QTableMetaData table = storageInput.getTable(); + QBackendMetaData backend = storageInput.getBackend(); + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + storageInput.getReference()); + + ///////////////////////////////////////////////////////////// + // s3 seems to do better w/o leading slashes, so, strip... // + ///////////////////////////////////////////////////////////// + if(fullPath.startsWith("/")) + { + fullPath = fullPath.substring(1); + } + + return fullPath; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InputStream getInputStream(StorageInput storageInput) throws QException + { + try + { + S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend(); + preAction(backend); + + AmazonS3 amazonS3 = getS3Utils().getAmazonS3(); + String fullPath = getFullPath(storageInput); + GetObjectRequest getObjectRequest = new GetObjectRequest(backend.getBucketName(), fullPath); + S3Object s3Object = amazonS3.getObject(getObjectRequest); + S3ObjectInputStream objectContent = s3Object.getObjectContent(); + + return (objectContent); + } + catch(Exception e) + { + throw (new QException("Exception getting s3 input stream for file", e)); + } + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java index 652913f7..2475b6b5 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java @@ -50,19 +50,6 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData - /******************************************************************************* - ** Fluent setter for backendType - ** - *******************************************************************************/ - @Override - public S3BackendMetaData withBackendType(String backendType) - { - setBackendType(backendType); - return this; - } - - - /******************************************************************************* ** Getter for bucketName ** diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStream.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStream.java new file mode 100644 index 00000000..f67493ee --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStream.java @@ -0,0 +1,205 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.utils; + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectResult; +import com.amazonaws.services.s3.model.UploadPartRequest; +import com.amazonaws.services.s3.model.UploadPartResult; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** OutputStream implementation that knows how to stream data into a new S3 file. + ** + ** This will be done using a multipart-upload if the contents are > 5MB - else + ** just a 1-time-call to PutObject + *******************************************************************************/ +public class S3UploadOutputStream extends OutputStream +{ + private static final QLogger LOG = QLogger.getLogger(S3UploadOutputStream.class); + + private final AmazonS3 amazonS3; + private final String bucketName; + private final String key; + + private byte[] buffer = new byte[5 * 1024 * 1024]; + private int offset = 0; + + private InitiateMultipartUploadResult initiateMultipartUploadResult = null; + private List uploadPartResultList = null; + + private boolean isClosed = false; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public S3UploadOutputStream(AmazonS3 amazonS3, String bucketName, String key) + { + this.amazonS3 = amazonS3; + this.bucketName = bucketName; + this.key = key; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void write(int b) throws IOException + { + buffer[offset] = (byte) b; + offset++; + + uploadIfNeeded(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void uploadIfNeeded() + { + if(offset == buffer.length) + { + ////////////////////////////////////////// + // start or continue a multipart upload // + ////////////////////////////////////////// + if(initiateMultipartUploadResult == null) + { + LOG.info("Initiating a multipart upload", logPair("key", key)); + initiateMultipartUploadResult = amazonS3.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucketName, key)); + uploadPartResultList = new ArrayList<>(); + } + + LOG.info("Uploading a part", logPair("key", key), logPair("partNumber", uploadPartResultList.size() + 1)); + UploadPartRequest uploadPartRequest = new UploadPartRequest() + .withUploadId(initiateMultipartUploadResult.getUploadId()) + .withPartNumber(uploadPartResultList.size() + 1) + .withInputStream(new ByteArrayInputStream(buffer)) + .withBucketName(bucketName) + .withKey(key) + .withPartSize(buffer.length); + + uploadPartResultList.add(amazonS3.uploadPart(uploadPartRequest)); + + ////////////////// + // reset buffer // + ////////////////// + offset = 0; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void write(byte[] b, int off, int len) throws IOException + { + int bytesToWrite = len; + + while(bytesToWrite > buffer.length - offset) + { + int size = buffer.length - offset; + System.arraycopy(b, off, buffer, offset, size); + offset = buffer.length; + uploadIfNeeded(); + off += size; + bytesToWrite -= size; + } + + int size = len - off; + System.arraycopy(b, off, buffer, offset, size); + offset += size; + uploadIfNeeded(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void close() throws IOException + { + if(isClosed) + { + LOG.debug("Redundant call to close an already-closed S3UploadOutputStream. Returning with noop.", logPair("key", key)); + return; + } + + if(initiateMultipartUploadResult != null) + { + if(offset > 0) + { + ////////////////////////////////////////////////// + // if there's a final part to upload, do it now // + ////////////////////////////////////////////////// + LOG.info("Uploading a part", logPair("key", key), logPair("isFinalPart", true), logPair("partNumber", uploadPartResultList.size() + 1)); + UploadPartRequest uploadPartRequest = new UploadPartRequest() + .withUploadId(initiateMultipartUploadResult.getUploadId()) + .withPartNumber(uploadPartResultList.size() + 1) + .withInputStream(new ByteArrayInputStream(buffer, 0, offset)) + .withBucketName(bucketName) + .withKey(key) + .withPartSize(offset); + uploadPartResultList.add(amazonS3.uploadPart(uploadPartRequest)); + } + + CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest() + .withUploadId(initiateMultipartUploadResult.getUploadId()) + .withPartETags(uploadPartResultList) + .withBucketName(bucketName) + .withKey(key); + CompleteMultipartUploadResult completeMultipartUploadResult = amazonS3.completeMultipartUpload(completeMultipartUploadRequest); + } + else + { + LOG.info("Putting object (non-multipart)", logPair("key", key), logPair("length", offset)); + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentLength(offset); + PutObjectResult putObjectResult = amazonS3.putObject(bucketName, key, new ByteArrayInputStream(buffer, 0, offset), objectMetadata); + } + + isClosed = true; + } + +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index 68c99d28..acad9100 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -40,6 +40,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; @@ -63,16 +64,16 @@ public class TestUtils public static final String BACKEND_NAME_S3 = "s3"; public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; public static final String BACKEND_NAME_MOCK = "mock"; - public static final String BACKEND_NAME_MEMORY = "memory"; + public static final String BACKEND_NAME_MEMORY = "memory"; public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json"; public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv"; public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob"; - public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive"; + public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive"; public static final String TABLE_NAME_PERSON_S3 = "person-s3"; public static final String TABLE_NAME_BLOB_S3 = "s3-blob"; public static final String TABLE_NAME_PERSON_MOCK = "person-mock"; - public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; + public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed"; public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter"; @@ -403,7 +404,7 @@ public class TestUtils public static QBackendMetaData defineMockBackend() { return (new QBackendMetaData() - .withBackendType("mock") + .withBackendType(MockBackendModule.class) .withName(BACKEND_NAME_MOCK)); } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageActionTest.java new file mode 100644 index 00000000..be47210b --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageActionTest.java @@ -0,0 +1,63 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.local.actions; + + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for FilesystemStorageAction + *******************************************************************************/ +class FilesystemStorageActionTest extends FilesystemActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws Exception + { + String data = "Hellooo, Storage."; + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withReference("test.txt"); + + OutputStream outputStream = new StorageAction().createOutputStream(storageInput); + outputStream.write(data.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + InputStream inputStream = new StorageAction().getInputStream(storageInput); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + inputStream.transferTo(byteArrayOutputStream); + + assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8)); + + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3StorageActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3StorageActionTest.java new file mode 100644 index 00000000..4df407ee --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3StorageActionTest.java @@ -0,0 +1,68 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for FilesystemStorageAction + *******************************************************************************/ +public class S3StorageActionTest extends BaseS3Test +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws Exception + { + String data = "Hellooo, Storage."; + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_BLOB_S3).withReference("test.txt"); + + ///////////////////////////////////////////////////////////////////////// + // work directly w/ s3 action class here, so we can set s3 utils in it // + ///////////////////////////////////////////////////////////////////////// + S3StorageAction s3StorageAction = new S3StorageAction(); + s3StorageAction.setS3Utils(getS3Utils()); + OutputStream outputStream = s3StorageAction.createOutputStream(storageInput); + outputStream.write(data.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + InputStream inputStream = s3StorageAction.getInputStream(storageInput); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + inputStream.transferTo(byteArrayOutputStream); + + assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8)); + + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStreamTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStreamTest.java new file mode 100644 index 00000000..96d43bce --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStreamTest.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.utils; + + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for S3UploadOutputStream + *******************************************************************************/ +class S3UploadOutputStreamTest extends BaseS3Test +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws IOException + { + String bucketName = BaseS3Test.BUCKET_NAME; + String key = "uploader-tests/" + Instant.now().toString() + ".txt"; + + // S3UploadOutputStream outputStream = new S3UploadOutputStream(amazonS3, bucketName, key); + // FileOutputStream outputStream = new FileOutputStream("/tmp/file.json"); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + outputStream.write("[\n1".getBytes(StandardCharsets.UTF_8)); + for(int i = 2; i <= 1_000_000; i++) + { + outputStream.write((",\n" + i).getBytes(StandardCharsets.UTF_8)); + } + outputStream.write("\n]\n".getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + S3UploadOutputStream s3UploadOutputStream = new S3UploadOutputStream(getS3Utils().getAmazonS3(), bucketName, key); + s3UploadOutputStream.write(outputStream.toByteArray(), 0, 5 * 1024 * 1024); + s3UploadOutputStream.write(outputStream.toByteArray(), 0, 3 * 1024 * 1024); + s3UploadOutputStream.write(outputStream.toByteArray(), 0, 3 * 1024 * 1024); + s3UploadOutputStream.close(); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java index b1bffa96..5a84d73b 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.module.rdbms.actions.AbstractRDBMSAction; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSAggregateAction; @@ -55,7 +56,10 @@ public class RDBMSBackendModule implements QBackendModuleInterface { private static final QLogger LOG = QLogger.getLogger(RDBMSBackendModule.class); - + static + { + QBackendModuleDispatcher.registerBackendModule(new RDBMSBackendModule()); + } /******************************************************************************* ** Method where a backend module must be able to provide its type (name). diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index 7e858982..e3c60d0d 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -857,6 +857,17 @@ public abstract class AbstractRDBMSAction + /******************************************************************************* + ** Make it easy (e.g., for tests) to turn on logging of SQL + *******************************************************************************/ + public static void setLogSQL(boolean on, boolean doReformat, String loggerOrSystemOut) + { + setLogSQL(on); + setLogSQLOutput(loggerOrSystemOut); + setLogSQLReformat(doReformat); + } + + /******************************************************************************* ** Make it easy (e.g., for tests) to turn on logging of SQL *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java index ccf5dd86..2570f6ea 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -46,6 +47,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; @@ -61,6 +63,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; public class TestUtils { public static final String DEFAULT_BACKEND_NAME = "default"; + public static final String MEMORY_BACKEND_NAME = "memory"; public static final String TABLE_NAME_PERSON = "personTable"; public static final String TABLE_NAME_PERSONAL_ID_CARD = "personalIdCard"; @@ -107,6 +110,7 @@ public class TestUtils { QInstance qInstance = new QInstance(); qInstance.addBackend(defineBackend()); + qInstance.addBackend(defineMemoryBackend()); qInstance.addTable(defineTablePerson()); qInstance.addPossibleValueSource(definePvsPerson()); qInstance.addTable(defineTablePersonalIdCard()); @@ -118,6 +122,18 @@ public class TestUtils + /******************************************************************************* + ** Define the in-memory backend used in standard tests + *******************************************************************************/ + public static QBackendMetaData defineMemoryBackend() + { + return new QBackendMetaData() + .withName(MEMORY_BACKEND_NAME) + .withBackendType(MemoryBackendModule.class); + } + + + /******************************************************************************* ** Define the authentication used in standard tests - using 'mock' type. ** @@ -243,7 +259,7 @@ public class TestUtils .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId")) .withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_ORDER_LINE).withJoinName("orderJoinOrderLine")) .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ITEM).withJoinPath(List.of("orderJoinOrderLine", "orderLineJoinItem"))) - .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER_INSTRUCTIONS).withJoinPath(List.of("orderJoinCurrentOrderInstructions"))) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER_INSTRUCTIONS).withJoinPath(List.of("orderJoinCurrentOrderInstructions")).withLabel("Current Order Instructions")) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE)) .withField(new QFieldMetaData("billToPersonId", QFieldType.INTEGER).withBackendName("bill_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) .withField(new QFieldMetaData("shipToPersonId", QFieldType.INTEGER).withBackendName("ship_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java index e49bc014..9e3be3ed 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/reporting/GenerateReportActionRDBMSTest.java @@ -23,25 +23,52 @@ package com.kingsrook.qqq.backend.module.rdbms.reporting; import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableDefinition; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableFunction; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryStorageAction; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RenderSavedReportMetaDataProducer; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.LocalMacDevUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest; +import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -196,6 +223,220 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest + /******************************************************************************* + ** + *******************************************************************************/ + private RunProcessOutput runSavedReport(SavedReport savedReport, ReportFormatPossibleValueEnum reportFormat) throws Exception + { + savedReport.setLabel("Test Report"); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + + if(QContext.getQInstance().getTable(SavedReport.TABLE_NAME) == null) + { + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + } + + QRecord savedReportRecord = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(savedReport)).getRecords().get(0); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName(RenderSavedReportMetaDataProducer.NAME); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.setCallback(QProcessCallbackFactory.forRecord(savedReportRecord)); + input.addValue("reportFormat", reportFormat); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(input); + return (runProcessOutput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List runSavedReportForCSV(SavedReport savedReport) throws Exception + { + RunProcessOutput runProcessOutput = runSavedReport(savedReport, ReportFormatPossibleValueEnum.CSV); + + String storageTableName = runProcessOutput.getValueString("storageTableName"); + String storageReference = runProcessOutput.getValueString("storageReference"); + InputStream inputStream = new MemoryStorageAction().getInputStream(new StorageInput(storageTableName).withReference(storageReference)); + + return (IOUtils.readLines(inputStream, StandardCharsets.UTF_8)); + } + + + + /******************************************************************************* + ** in here, by potentially ambiguous, we mean where there are possible joins + ** between the order and orderInstructions tables. + *******************************************************************************/ + @Test + void testSavedReportWithPotentiallyAmbiguousExposedJoinSelections() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("storeId") + .withColumn("orderInstructions.instructions"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()))); + + assertEquals(""" + "Id","Store","Order Instructions: Instructions" + """.trim(), lines.get(0)); + assertEquals(""" + "1","Q-Mart","order 1 v2" + """.trim(), lines.get(1)); + } + + + + /******************************************************************************* + ** in here, by potentially ambiguous, we mean where there are possible joins + ** between the order and orderInstructions tables. + *******************************************************************************/ + @Test + void testSavedReportWithPotentiallyAmbiguousExposedJoinSelectedAndOrdered() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("storeId") + .withColumn("orderInstructions.instructions"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter() + .withOrderBy(new QFilterOrderBy("orderInstructions.id", false)) + ))); + + assertEquals(""" + "Id","Store","Order Instructions: Instructions" + """.trim(), lines.get(0)); + assertEquals(""" + "8","QDepot","order 8 v1" + """.trim(), lines.get(1)); + } + + + + /******************************************************************************* + ** in here, by potentially ambiguous, we mean where there are possible joins + ** between the order and orderInstructions tables. + *******************************************************************************/ + @Test + void testSavedReportWithPotentiallyAmbiguousExposedJoinCriteria() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("storeId"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter() + .withCriteria(new QFilterCriteria("orderInstructions.instructions", QCriteriaOperator.CONTAINS, "v3")) + ))); + + assertEquals(""" + "Id","Store" + """.trim(), lines.get(0)); + assertEquals(""" + "2","Q-Mart" + """.trim(), lines.get(1)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSavedReportWithExposedJoinMultipleTablesAwaySelected() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("storeId") + .withColumn("item.description"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter()))); + + assertEquals(""" + "Id","Store","Item: Description" + """.trim(), lines.get(0)); + assertEquals(""" + "1","Q-Mart","Q-Mart Item 1" + """.trim(), lines.get(1)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSavedReportWithExposedJoinMultipleTablesAwayAsCriteria() throws Exception + { + List lines = runSavedReportForCSV(new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("storeId"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter() + .withCriteria(new QFilterCriteria("item.description", QCriteriaOperator.CONTAINS, "Item 7")) + ))); + + assertEquals(""" + "Id","Store" + """.trim(), lines.get(0)); + assertEquals(""" + "6","QDepot" + """.trim(), lines.get(1)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSavedReportWithPivotsFromJoinTable() throws Exception + { + SavedReport savedReport = new SavedReport() + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("item.storeId") + .withColumn("item.description"))) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withPivotTableJson(JsonUtils.toJson(new PivotTableDefinition() + .withRow(new PivotTableGroupBy().withFieldName("item.storeId")) + .withValue(new PivotTableValue().withFieldName("item.description").withFunction(PivotTableFunction.COUNT)))); + + ////////////////////////////////////////////// + // make sure we can render xlsx w/o a crash // + ////////////////////////////////////////////// + RunProcessOutput runProcessOutput = runSavedReport(savedReport, ReportFormatPossibleValueEnum.XLSX); + String storageTableName = runProcessOutput.getValueString("storageTableName"); + String storageReference = runProcessOutput.getValueString("storageReference"); + InputStream inputStream = new MemoryStorageAction().getInputStream(new StorageInput(storageTableName).withReference(storageReference)); + + String path = "/tmp/pivot.xlsx"; + inputStream.transferTo(new FileOutputStream(path)); + // LocalMacDevUtils.mayOpenFiles = true; + LocalMacDevUtils.openFile(path); + + /////////////////////////////////////////////////////// + // render as csv too - and assert about those values // + /////////////////////////////////////////////////////// + List csv = runSavedReportForCSV(savedReport); + System.out.println(StringUtils.join("\n", csv)); + assertEquals(""" + "Store","Count Of Item: Description\"""", csv.get(0)); + assertEquals(""" + "Q-Mart","4\"""", csv.get(1)); + assertEquals(""" + "Totals","11\"""", csv.get(4)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -204,9 +445,12 @@ public class GenerateReportActionRDBMSTest extends RDBMSActionTest ReportInput reportInput = new ReportInput(); QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); reportInput.setReportName(TEST_REPORT); - reportInput.setReportFormat(ReportFormat.CSV); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - reportInput.setReportOutputStream(outputStream); + + reportInput.setReportDestination(new ReportDestination() + .withReportFormat(ReportFormat.CSV) + .withReportOutputStream(outputStream)); + new GenerateReportAction().execute(reportInput); return (outputStream.toString()); } diff --git a/qqq-dev-tools/bin/xbar-circleci-latest.sh b/qqq-dev-tools/bin/xbar-circleci-latest.sh index 0b6f92c7..4a39eb8d 100755 --- a/qqq-dev-tools/bin/xbar-circleci-latest.sh +++ b/qqq-dev-tools/bin/xbar-circleci-latest.sh @@ -14,12 +14,15 @@ . ~/.bashrc . $QQQ_DEV_TOOLS_DIR/.env -FILE=/tmp/cci.$$ +DIR=/tmp/xbar-circleci-latest +mkdir -p $DIR +FILE=$DIR/cci.$$ JQ=/opt/homebrew/bin/jq -curl -s -H "Circle-Token: ${CIRCLE_TOKEN}" "https://circleci.com/api/v1.1/recent-builds?limit=10&shallow=true" > $FILE +curl -s -H "Circle-Token: ${CIRCLE_TOKEN}" "https://circleci.com/api/v1.1/recent-builds?limit=50&shallow=true" > $FILE NOW=$(date +%s) needPipe=0 +displayedAny=0 checkBuild() { @@ -38,11 +41,11 @@ checkBuild() fi endDate=$($JQ ".[$i].stop_time" < $FILE | sed 's/"//g;s/null//;') - curl $avatarUrl > /tmp/avatar.jpg - sips -s dpiHeight 96 -s dpiWidth 96 /tmp/avatar.jpg -o /tmp/avatar-96dpi.jpg > /dev/null - sips -z 20 20 /tmp/avatar-96dpi.jpg -o /tmp/avatar-20.jpg > /dev/null - base64 -i /tmp/avatar-20.jpg > /tmp/avatar.b64 - avatarB64=$(cat /tmp/avatar.b64) + curl $avatarUrl > $DIR/avatar.jpg + sips -s dpiHeight 96 -s dpiWidth 96 $DIR/avatar.jpg -o $DIR/avatar-96dpi.jpg > /dev/null + sips -z 20 20 $DIR/avatar-96dpi.jpg -o $DIR/avatar-20.jpg > /dev/null + base64 -i $DIR/avatar-20.jpg > $DIR/avatar.b64 + avatarB64=$(cat $DIR/avatar.b64) shortRepo="$repo" case $repo in @@ -50,8 +53,10 @@ checkBuild() qqq-frontend-core) shortRepo="fc";; qqq-frontend-material-dashboard) shortRepo="qfmd";; ColdTrack-Live) shortRepo="ctl";; - ColdTrack-Live-Scripts) shortRepo="cls";; - Infoplus-Scripts) shortRepo="ips";; + ## ColdTrack-Live-Scripts) shortRepo="cls";; + ## Infoplus-Scripts) shortRepo="ips";; + ColdTrack-Live-Scripts) return;; + Infoplus-Scripts) return;; esac timestamp=$(date -j -f "%Y-%m-%dT%H:%M:%S%z" $(echo "$startDate" | sed 's/\....Z/+0000/') +%s) @@ -93,7 +98,9 @@ checkBuild() #################################################################################################### ## if this is the 1st build, or it was less than some-short-time ago, then put it in the menu bar ## #################################################################################################### - if [ $index -lt 1 -o $seconds -lt 600 ]; then + if [ $displayedAny -eq 0 -o $seconds -lt 600 -o "$buildStatus" == "running" ]; then + + displayedAny=1 ########################################################################################### ## put a pipe (unicode special pipe, to not break things) before all but the first build ## @@ -115,14 +122,18 @@ checkBuild() details="---" details="$details\n🔄 Refresh | refresh=true" -for i in $(seq 0 9); do +for i in $(seq 0 19); do checkBuild $i done +if [ $displayedAny -eq 0 ]; then + echo "🤷" +fi + ## echo "@$(date +%M:%S)" echo echo -e "$details" -cp $FILE /tmp/cci-latest.json +cp $FILE $DIR/cci-latest.json rm $FILE diff --git a/qqq-language-support-javascript/src/test/java/com/kingsrook/qqq/languages/javascript/TestUtils.java b/qqq-language-support-javascript/src/test/java/com/kingsrook/qqq/languages/javascript/TestUtils.java index a7eab8b5..a094c1c9 100644 --- a/qqq-language-support-javascript/src/test/java/com/kingsrook/qqq/languages/javascript/TestUtils.java +++ b/qqq-language-support-javascript/src/test/java/com/kingsrook/qqq/languages/javascript/TestUtils.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticat import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; /******************************************************************************* @@ -74,7 +75,7 @@ public class TestUtils { return (new QBackendMetaData() .withName(DEFAULT_BACKEND_NAME) - .withBackendType("memory")); + .withBackendType(MemoryBackendModule.class)); } diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java index 56444250..84137b7d 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java @@ -1000,6 +1000,7 @@ public class ApiImplementation ApiProcessInput apiProcessInput = apiProcessMetaData.getInput(); if(apiProcessInput != null) { + processProcessInputFields(paramMap, badRequestMessages, runProcessInput, apiProcessInput.getPathParams()); processProcessInputFields(paramMap, badRequestMessages, runProcessInput, apiProcessInput.getQueryStringParams()); processProcessInputFields(paramMap, badRequestMessages, runProcessInput, apiProcessInput.getFormParams()); processProcessInputFields(paramMap, badRequestMessages, runProcessInput, apiProcessInput.getObjectBodyParams()); @@ -1143,7 +1144,10 @@ public class ApiImplementation ApiProcessOutputInterface output = apiProcessMetaData.getOutput(); if(output != null) { - return (new HttpApiResponse(output.getSuccessStatusCode(runProcessInput, runProcessOutput), output.getOutputForProcess(runProcessInput, runProcessOutput))); + Serializable outputForProcess = output.getOutputForProcess(runProcessInput, runProcessOutput); + HttpApiResponse httpApiResponse = new HttpApiResponse(output.getSuccessStatusCode(runProcessInput, runProcessOutput), outputForProcess); + output.customizeHttpApiResponse(httpApiResponse, runProcessInput, runProcessOutput); + return httpApiResponse; } else { diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java index a4ad7378..35457cdc 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java @@ -194,6 +194,8 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction neededTableSchemas = new HashSet<>(); @@ -314,40 +316,40 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction parameters = new ArrayList<>(); + ApiProcessInput apiProcessInput = apiProcessMetaData.getInput(); + ApiProcessInputFieldsContainer pathParams = apiProcessInput.getPathParams(); + if(pathParams != null) + { + for(QFieldMetaData field : CollectionUtils.nonNullList(pathParams.getFields())) + { + parameters.add(processFieldToParameter(apiInstanceMetaData, field).withIn("path")); + } + } + + parameters.add(new Parameter() .withName("jobId") .withIn("path") .withRequired(true) .withDescription("Id of the job, as returned by the API call that started it.") - .withSchema(new Schema().withType("string").withFormat("uuid")) - )); + .withSchema(new Schema().withType("string").withFormat("uuid"))); + + //////////////////////////////////////////////////////// + // add the async input for optionally-async processes // + //////////////////////////////////////////////////////// + methodForProcess.setParameters(parameters); ////////////////////////////////// // build all possible responses // @@ -1126,7 +1159,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction. + */ + +package com.kingsrook.qqq.api.implementations.savedreports; + + +import com.kingsrook.qqq.api.model.metadata.processes.PreRunApiProcessCustomizer; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; + + +/******************************************************************************* + ** API-Customizer for the RenderSavedReport process + *******************************************************************************/ +public class RenderSavedReportProcessApiCustomizer implements PreRunApiProcessCustomizer +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void preApiRun(RunProcessInput runProcessInput) throws QException + { + Integer reportId = runProcessInput.getValueInteger("reportId"); + if(reportId != null) + { + QRecord record = new GetAction().executeForRecord(new GetInput(SavedReport.TABLE_NAME).withPrimaryKey(reportId)); + if(record == null) + { + throw (new QNotFoundException("Report Id " + reportId + " was not found.")); + } + + runProcessInput.setCallback(QProcessCallbackFactory.forPrimaryKey("id", reportId)); + } + } + +} diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/implementations/savedreports/RenderSavedReportProcessApiMetaDataEnricher.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/implementations/savedreports/RenderSavedReportProcessApiMetaDataEnricher.java new file mode 100644 index 00000000..528adfe8 --- /dev/null +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/implementations/savedreports/RenderSavedReportProcessApiMetaDataEnricher.java @@ -0,0 +1,105 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.api.implementations.savedreports; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData; +import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaDataContainer; +import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessCustomizers; +import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessInput; +import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessInputFieldsContainer; +import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessMetaData; +import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessMetaDataContainer; +import com.kingsrook.qqq.api.model.openapi.ExampleWithListValue; +import com.kingsrook.qqq.api.model.openapi.ExampleWithSingleValue; +import com.kingsrook.qqq.api.model.openapi.HttpMethod; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormatPossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; + + +/******************************************************************************* + ** Class that helps prepare the RenderSavedReport process for use in an API + *******************************************************************************/ +public class RenderSavedReportProcessApiMetaDataEnricher +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public static ApiProcessMetaData setupProcessForApi(QProcessMetaData process, String apiName, String initialApiVersion) + { + ApiProcessMetaDataContainer apiProcessMetaDataContainer = ApiProcessMetaDataContainer.ofOrWithNew(process); + + ApiProcessInput input = new ApiProcessInput() + .withPathParams(new ApiProcessInputFieldsContainer() + .withField(new QFieldMetaData("reportId", QFieldType.INTEGER) + .withIsRequired(true) + .withSupplementalMetaData(newDefaultApiFieldMetaData("Saved Report Id", 1701)))) + .withQueryStringParams(new ApiProcessInputFieldsContainer() + .withField(new QFieldMetaData("reportFormat", QFieldType.STRING) + .withIsRequired(true) + .withPossibleValueSourceName(ReportFormatPossibleValueEnum.NAME) + .withSupplementalMetaData(newDefaultApiFieldMetaData("Requested file format", "XLSX")))); + // todo (when implemented) - probably a JSON doc w/ input values. + + RenderSavedReportProcessApiProcessOutput output = new RenderSavedReportProcessApiProcessOutput(); + + ApiProcessMetaData apiProcessMetaData = new ApiProcessMetaData() + .withInitialVersion(initialApiVersion) + .withCustomizer(ApiProcessCustomizers.PRE_RUN.getRole(), new QCodeReference(RenderSavedReportProcessApiCustomizer.class)) + .withAsyncMode(ApiProcessMetaData.AsyncMode.OPTIONAL) + .withMethod(HttpMethod.GET) + .withInput(input) + .withOutput(output); + + apiProcessMetaDataContainer.withApiProcessMetaData(apiName, apiProcessMetaData); + + return (apiProcessMetaData); + } + + + + /******************************************************************************* + ** todo - move to higher-level utility + *******************************************************************************/ + public static ApiFieldMetaDataContainer newDefaultApiFieldMetaData(String description, Serializable example) + { + ApiFieldMetaData defaultApiFieldMetaData = new ApiFieldMetaData().withDescription(description); + ApiFieldMetaDataContainer apiFieldMetaDataContainer = new ApiFieldMetaDataContainer().withDefaultApiFieldMetaData(defaultApiFieldMetaData); + if(example instanceof List) + { + defaultApiFieldMetaData.withExample(new ExampleWithListValue().withValue((List) example)); + } + else + { + defaultApiFieldMetaData.withExample(new ExampleWithSingleValue().withValue(example)); + } + + return (apiFieldMetaDataContainer); + } + +} diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/implementations/savedreports/RenderSavedReportProcessApiProcessOutput.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/implementations/savedreports/RenderSavedReportProcessApiProcessOutput.java new file mode 100644 index 00000000..f2429917 --- /dev/null +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/implementations/savedreports/RenderSavedReportProcessApiProcessOutput.java @@ -0,0 +1,141 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.api.implementations.savedreports; + + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; +import com.kingsrook.qqq.api.model.actions.HttpApiResponse; +import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessOutputInterface; +import com.kingsrook.qqq.api.model.openapi.Content; +import com.kingsrook.qqq.api.model.openapi.Response; +import com.kingsrook.qqq.api.model.openapi.Schema; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import org.eclipse.jetty.http.HttpStatus; + + +/******************************************************************************* + ** api process output specifier for the RenderSavedReport process + *******************************************************************************/ +public class RenderSavedReportProcessApiProcessOutput implements ApiProcessOutputInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Serializable getOutputForProcess(RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) throws QException + { + ////////////////////////////////////////////////////////////////// + // we don't use output like this - see customizeHttpApiResponse // + ////////////////////////////////////////////////////////////////// + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void customizeHttpApiResponse(HttpApiResponse httpApiResponse, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // we don't need anyone else to format our response - assume that we've done so ourselves. // + ///////////////////////////////////////////////////////////////////////////////////////////// + httpApiResponse.setNeedsFormattedAsJson(false); + + ///////////////////////////////////////////// + // set content type based on report format // + ///////////////////////////////////////////// + ReportFormat reportFormat = ReportFormat.fromString(runProcessOutput.getValueString("reportFormat")); + httpApiResponse.setContentType(reportFormat.getMimeType()); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // get an input stream from the backend where the report content is stored - send that down to the caller // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + String storageTableName = runProcessOutput.getValueString("storageTableName"); + String storageReference = runProcessOutput.getValueString("storageReference"); + httpApiResponse.setInputStream(new StorageAction().getInputStream(new StorageInput(storageTableName).withReference(storageReference))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public HttpStatus.Code getSuccessStatusCode(RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) + { + return (HttpStatus.Code.OK); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Map getSpecResponses(String apiName) + { + Map contentMap = new LinkedHashMap<>(); + contentMap.put(ReportFormat.JSON.getMimeType(), new Content() + .withSchema(new Schema() + .withDescription("JSON Report contents") + .withExample(""" + [ + {"id": 1, "name": "James"}, + {"id": 2, "name": "Jean-Luc"} + ] + """) + .withType("string") + .withFormat("text"))); + + contentMap.put(ReportFormat.CSV.getMimeType(), new Content() + .withSchema(new Schema() + .withDescription("CSV Report contents") + .withExample(""" + "id","name" + 1,"James" + 2,"Jean-Luc" + """) + .withType("string") + .withFormat("text"))); + + contentMap.put(ReportFormat.XLSX.getMimeType(), new Content() + .withSchema(new Schema() + .withDescription("Excel Report contents") + .withType("string") + .withFormat("binary"))); + + return Map.of(HttpStatus.Code.OK.getCode(), new Response() + .withDescription("Report contents in the requested format.") + .withContent(contentMap)); + } + +} diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java index cd99af77..99951958 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java @@ -316,6 +316,7 @@ public class QJavalinApiHandler ApiProcessInput input = apiProcessMetaData.getInput(); if(input != null) { + processProcessInputFieldsContainer(context, parameters, input.getPathParams(), Context::pathParam); processProcessInputFieldsContainer(context, parameters, input.getQueryStringParams(), Context::queryParam); processProcessInputFieldsContainer(context, parameters, input.getFormParams(), Context::formParam); @@ -347,9 +348,7 @@ public class QJavalinApiHandler ////////////////// QJavalinAccessLogger.logEndSuccess(); context.status(response.getStatusCode().getCode()); - String resultString = toJson(Objects.requireNonNullElse(response.getResponseBodyObject(), "")); - context.result(resultString); - storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); + handleProcessResponse(context, response, apiLog); } catch(Exception e) { @@ -360,6 +359,63 @@ public class QJavalinApiHandler + /******************************************************************************* + ** + *******************************************************************************/ + private static void handleProcessResponse(Context context, HttpApiResponse response, APILog apiLog) + { + if(response.getNeedsFormattedAsJson()) + { + ///////////////////////////////////////////////////////////////////////////////////////// + // if the response object says that we should format the response as json, then do so. // + ///////////////////////////////////////////////////////////////////////////////////////// + String resultString = toJson(Objects.requireNonNullElse(response.getResponseBodyObject(), "")); + context.result(resultString); + storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); + } + else + { + if(StringUtils.hasContent(response.getContentType())) + { + context.contentType(response.getContentType()); + } + + /////////////////////////////////////////////////////////////////////////////////// + // if there's an input stream in the response, just send that down to the client // + /////////////////////////////////////////////////////////////////////////////////// + if(response.getInputStream() != null) + { + context.result(response.getInputStream()); + storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody("Streamed result")); + } + else + { + //////////////////////////////////////////////////////////////////////////////////// + // else, try to return it raw - as byte[], or String, or as a converted-to-String // + //////////////////////////////////////////////////////////////////////////////////// + Serializable result = Objects.requireNonNullElse(response.getResponseBodyObject(), ""); + if(result instanceof byte[] ba) + { + context.result(ba); + storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody("Byte array of length: " + ba.length)); + } + else if(result instanceof String s) + { + context.result(s); + storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(s)); + } + else + { + String resultString = String.valueOf(result); + context.result(resultString); + storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); + } + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -381,9 +437,7 @@ public class QJavalinApiHandler ////////////////// QJavalinAccessLogger.logEndSuccess(); context.status(response.getStatusCode().getCode()); - String resultString = toJson(Objects.requireNonNullElse(response.getResponseBodyObject(), "")); - context.result(resultString); - storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); + handleProcessResponse(context, response, apiLog); } catch(Exception e) { diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/actions/HttpApiResponse.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/actions/HttpApiResponse.java index 5d3b1c66..8bedcda0 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/actions/HttpApiResponse.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/actions/HttpApiResponse.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.api.model.actions; +import java.io.InputStream; import java.io.Serializable; import org.eclipse.jetty.http.HttpStatus; @@ -35,6 +36,16 @@ public class HttpApiResponse private HttpStatus.Code statusCode; private Serializable responseBodyObject; + private String contentType; + + private InputStream inputStream; + + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // by default - QJavalinApiHandler will format the responseBodyObject as JSON. // + // set this field to false if you don't want it to do that (e.g., if your response is, say, a byte[]) // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + private boolean needsFormattedAsJson = true; + /******************************************************************************* @@ -119,4 +130,97 @@ public class HttpApiResponse return (this); } + + + /******************************************************************************* + ** Getter for needsFormattedAsJson + *******************************************************************************/ + public boolean getNeedsFormattedAsJson() + { + return (this.needsFormattedAsJson); + } + + + + /******************************************************************************* + ** Setter for needsFormattedAsJson + *******************************************************************************/ + public void setNeedsFormattedAsJson(boolean needsFormattedAsJson) + { + this.needsFormattedAsJson = needsFormattedAsJson; + } + + + + /******************************************************************************* + ** Fluent setter for needsFormattedAsJson + *******************************************************************************/ + public HttpApiResponse withNeedsFormattedAsJson(boolean needsFormattedAsJson) + { + this.needsFormattedAsJson = needsFormattedAsJson; + return (this); + } + + + + /******************************************************************************* + ** Getter for contentType + *******************************************************************************/ + public String getContentType() + { + return (this.contentType); + } + + + + /******************************************************************************* + ** Setter for contentType + *******************************************************************************/ + public void setContentType(String contentType) + { + this.contentType = contentType; + } + + + + /******************************************************************************* + ** Fluent setter for contentType + *******************************************************************************/ + public HttpApiResponse withContentType(String contentType) + { + this.contentType = contentType; + return (this); + } + + + /******************************************************************************* + ** Getter for inputStream + *******************************************************************************/ + public InputStream getInputStream() + { + return (this.inputStream); + } + + + + /******************************************************************************* + ** Setter for inputStream + *******************************************************************************/ + public void setInputStream(InputStream inputStream) + { + this.inputStream = inputStream; + } + + + + /******************************************************************************* + ** Fluent setter for inputStream + *******************************************************************************/ + public HttpApiResponse withInputStream(InputStream inputStream) + { + this.inputStream = inputStream; + return (this); + } + + } \ No newline at end of file diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessInput.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessInput.java index 83b05691..7109abbb 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessInput.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessInput.java @@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; *******************************************************************************/ public class ApiProcessInput { + private ApiProcessInputFieldsContainer pathParams; private ApiProcessInputFieldsContainer queryStringParams; private ApiProcessInputFieldsContainer formParams; private ApiProcessInputFieldsContainer recordBodyParams; @@ -44,6 +45,11 @@ public class ApiProcessInput *******************************************************************************/ public String getRecordIdsParamName() { + if(pathParams != null && pathParams.getRecordIdsField() != null) + { + return (pathParams.getRecordIdsField().getName()); + } + if(queryStringParams != null && queryStringParams.getRecordIdsField() != null) { return (queryStringParams.getRecordIdsField().getName()); @@ -217,4 +223,35 @@ public class ApiProcessInput return (this); } + + /******************************************************************************* + ** Getter for pathParams + *******************************************************************************/ + public ApiProcessInputFieldsContainer getPathParams() + { + return (this.pathParams); + } + + + + /******************************************************************************* + ** Setter for pathParams + *******************************************************************************/ + public void setPathParams(ApiProcessInputFieldsContainer pathParams) + { + this.pathParams = pathParams; + } + + + + /******************************************************************************* + ** Fluent setter for pathParams + *******************************************************************************/ + public ApiProcessInput withPathParams(ApiProcessInputFieldsContainer pathParams) + { + this.pathParams = pathParams; + return (this); + } + + } diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessOutputInterface.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessOutputInterface.java index 6e430aa4..1ee3e70d 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessOutputInterface.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessOutputInterface.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.api.model.metadata.processes; import java.io.Serializable; import java.util.Map; +import com.kingsrook.qqq.api.model.actions.HttpApiResponse; import com.kingsrook.qqq.api.model.openapi.Response; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; @@ -61,4 +62,13 @@ public interface ApiProcessOutputInterface .withDescription("Process has been successfully executed.") )); } + + /******************************************************************************* + ** + *******************************************************************************/ + default void customizeHttpApiResponse(HttpApiResponse httpApiResponse, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) throws QException + { + + } + } diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessUtils.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessUtils.java index dc75f7ad..b5cba76d 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessUtils.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessUtils.java @@ -34,9 +34,11 @@ import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; import com.kingsrook.qqq.backend.core.logging.LogPair; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; import org.apache.commons.lang.BooleanUtils; @@ -162,9 +164,18 @@ public class ApiProcessUtils *******************************************************************************/ public static String getProcessApiPath(QInstance qInstance, QProcessMetaData process, ApiProcessMetaData apiProcessMetaData, ApiInstanceMetaData apiInstanceMetaData) { + StringBuilder pathParams = new StringBuilder(); + if(ObjectUtils.ifCan(() -> CollectionUtils.nullSafeHasContents(apiProcessMetaData.getInput().getPathParams().getFields()))) + { + for(QFieldMetaData field : apiProcessMetaData.getInput().getPathParams().getFields()) + { + pathParams.append("/{").append(field.getName()).append("}"); + } + } + if(StringUtils.hasContent(apiProcessMetaData.getPath())) { - return apiProcessMetaData.getPath() + "/" + apiProcessMetaData.getApiProcessName(); + return apiProcessMetaData.getPath() + "/" + apiProcessMetaData.getApiProcessName() + pathParams; } else if(StringUtils.hasContent(process.getTableName())) { @@ -182,11 +193,11 @@ public class ApiProcessUtils } } } - return tablePathPart + "/" + apiProcessMetaData.getApiProcessName(); + return tablePathPart + "/" + apiProcessMetaData.getApiProcessName() + pathParams; } else { - return apiProcessMetaData.getApiProcessName(); + return apiProcessMetaData.getApiProcessName() + pathParams; } } diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/BaseTest.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/BaseTest.java index 2d7fd5b3..e77c0119 100644 --- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/BaseTest.java +++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/BaseTest.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.api; import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -56,7 +57,7 @@ public class BaseTest ** *******************************************************************************/ @BeforeEach - void baseBeforeEach() + void baseBeforeEach() throws QException { QContext.init(TestUtils.defineInstance(), new QSession()); } diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TestUtils.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TestUtils.java index 1cc32785..66cb753c 100644 --- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TestUtils.java +++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TestUtils.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.api; import java.util.List; import java.util.function.Consumer; +import com.kingsrook.qqq.api.implementations.savedreports.RenderSavedReportProcessApiMetaDataEnricher; import com.kingsrook.qqq.api.model.APIVersion; import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData; import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaDataContainer; @@ -45,6 +46,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; @@ -71,7 +73,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMeta import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; @@ -79,6 +84,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.Mem import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaUpdateStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.savedreports.RenderSavedReportMetaDataProducer; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -112,7 +118,7 @@ public class TestUtils /******************************************************************************* ** *******************************************************************************/ - public static QInstance defineInstance() + public static QInstance defineInstance() throws QException { QInstance qInstance = new QInstance(); @@ -133,6 +139,8 @@ public class TestUtils qInstance.setAuthentication(new Auth0AuthenticationMetaData().withType(QAuthenticationType.FULLY_ANONYMOUS).withName("anonymous")); + addSavedReports(qInstance); + qInstance.withSupplementalMetaData(new ApiInstanceMetaDataContainer() .withApiInstanceMetaData(new ApiInstanceMetaData() .withName(API_NAME) @@ -161,6 +169,18 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + private static void addSavedReports(QInstance qInstance) throws QException + { + qInstance.add(TablesPossibleValueSourceMetaDataProvider.defineTablesPossibleValueSource(qInstance)); + new SavedReportsMetaDataProvider().defineAll(qInstance, MEMORY_BACKEND_NAME, MEMORY_BACKEND_NAME, null); + RenderSavedReportProcessApiMetaDataEnricher.setupProcessForApi(qInstance.getProcess(RenderSavedReportMetaDataProducer.NAME), API_NAME, V2022_Q4); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -531,6 +551,19 @@ public class TestUtils } + /******************************************************************************* + ** + *******************************************************************************/ + public static Integer insertSavedReport(SavedReport savedReport) throws QException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(SavedReport.TABLE_NAME); + insertInput.setRecords(List.of(savedReport.toQRecord())); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + return insertOutput.getRecords().get(0).getValueInteger("id"); + } + + /******************************************************************************* ** diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerPermissionsTest.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerPermissionsTest.java index 680c850d..c884d47c 100644 --- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerPermissionsTest.java +++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerPermissionsTest.java @@ -25,7 +25,6 @@ package com.kingsrook.qqq.api.javalin; import com.kingsrook.qqq.api.BaseTest; import com.kingsrook.qqq.api.TestUtils; import com.kingsrook.qqq.api.actions.ApiImplementation; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; @@ -61,7 +60,7 @@ class QJavalinApiHandlerPermissionsTest extends BaseTest ** *******************************************************************************/ @BeforeAll - static void beforeAll() throws QInstanceValidationException + static void beforeAll() throws Exception { QInstance qInstance = TestUtils.defineInstance(); diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java index f495f51c..20fc10b1 100644 --- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java +++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java @@ -36,7 +36,6 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; @@ -51,6 +50,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.javalin.QJavalinImplementation; @@ -66,6 +68,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.api.TestUtils.insertPersonRecord; +import static com.kingsrook.qqq.api.TestUtils.insertSavedReport; import static com.kingsrook.qqq.api.TestUtils.insertSimpsons; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertArrayEquals; @@ -95,7 +98,7 @@ class QJavalinApiHandlerTest extends BaseTest ** *******************************************************************************/ @BeforeAll - static void beforeAll() throws QInstanceValidationException + static void beforeAll() throws Exception { QInstance qInstance = TestUtils.defineInstance(); @@ -1404,6 +1407,51 @@ class QJavalinApiHandlerTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetProcessRenderSavedReport() throws QException + { + insertSimpsons(); + Integer reportId = insertSavedReport(new SavedReport() + .withLabel("Person Report") + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withQueryFilterJson(JsonUtils.toJson(new QQueryFilter())) + .withColumnsJson(JsonUtils.toJson(new ReportColumns() + .withColumn("id") + .withColumn("firstName") + .withColumn("lastName")))); + + HttpResponse response = Unirest.get(BASE_URL + "/api/" + VERSION + "/savedReport/renderSavedReport/" + reportId + "?reportFormat=CSV").asString(); + assertEquals(HttpStatus.OK_200, response.getStatus()); + assertThat(response.getHeaders().getFirst("content-type")).contains("csv"); + assertEquals(""" + "Id","First Name","Last Name" + "1","Homer","Simpson" + "2","Marge","Simpson" + "3","Bart","Simpson" + "4","Lisa","Simpson" + "5","Maggie","Simpson" + """, response.getBody()); + + response = Unirest.get(BASE_URL + "/api/" + VERSION + "/savedReport/renderSavedReport/" + reportId + "?reportFormat=JSON").asString(); + assertEquals(HttpStatus.OK_200, response.getStatus()); + assertThat(response.getHeaders().getFirst("content-type")).contains("json"); + JSONArray jsonArray = new JSONArray(response.getBody()); + assertEquals(5, jsonArray.length()); + assertThat(jsonArray.getJSONObject(0).toMap()) + .hasFieldOrPropertyWithValue("id", 1) + .hasFieldOrPropertyWithValue("firstName", "Homer") + .hasFieldOrPropertyWithValue("lastName", "Simpson"); + + response = Unirest.get(BASE_URL + "/api/" + VERSION + "/savedReport/renderSavedReport/" + reportId + "?reportFormat=XLSX").asString(); + assertEquals(HttpStatus.OK_200, response.getStatus()); + assertThat(response.getHeaders().getFirst("content-type")).contains("openxmlformats-officedocument.spreadsheetml"); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 6b674b09..9fa28ea8 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -33,6 +33,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -77,6 +78,7 @@ import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; @@ -427,7 +429,17 @@ public class QJavalinImplementation QSession session = authenticationModule.createSession(qInstance, authContext); context.cookie(SESSION_UUID_COOKIE_NAME, session.getUuid(), SESSION_COOKIE_AGE); - context.result(JsonUtils.toJson(MapBuilder.of("uuid", session.getUuid()))); + + Map resultMap = new HashMap<>(); + resultMap.put("uuid", session.getUuid()); + + if(session.getValuesForFrontend() != null) + { + LinkedHashMap valuesForFrontend = new LinkedHashMap<>(session.getValuesForFrontend()); + resultMap.put("values", valuesForFrontend); + } + + context.result(JsonUtils.toJson(resultMap)); } catch(Exception e) { @@ -1492,10 +1504,11 @@ public class QJavalinImplementation setupSession(context, exportInput); exportInput.setTableName(tableName); - exportInput.setReportFormat(reportFormat); String filename = optionalFilename.orElse(tableName + "." + reportFormat.toString().toLowerCase(Locale.ROOT)); - exportInput.setFilename(filename); + exportInput.withReportDestination(new ReportDestination() + .withReportFormat(reportFormat) + .withFilename(filename)); Integer limit = QJavalinUtils.integerQueryParam(context, "limit"); exportInput.setLimit(limit); @@ -1526,7 +1539,7 @@ public class QJavalinImplementation UnsafeFunction preAction = (PipedOutputStream pos) -> { - exportInput.setReportOutputStream(pos); + exportInput.getReportDestination().setReportOutputStream(pos); ExportAction exportAction = new ExportAction(); exportAction.preExecute(exportInput); @@ -1908,4 +1921,12 @@ public class QJavalinImplementation MILLIS_BETWEEN_HOT_SWAPS = millisBetweenHotSwaps; } + + /******************************************************************************* + ** + *******************************************************************************/ + public static long getStartTimeMillis() + { + return (startTime); + } } diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java index 941c9f24..d4238f64 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java @@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; @@ -60,12 +61,14 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState; import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -203,10 +206,12 @@ public class QJavalinProcessHandler QJavalinImplementation.setupSession(context, reportInput); PermissionsHelper.checkReportPermissionThrowing(reportInput, reportName); - reportInput.setReportFormat(reportFormat); reportInput.setReportName(reportName); reportInput.setInputValues(null); // todo! - reportInput.setFilename(filename); + + reportInput.setReportDestination(new ReportDestination() + .withReportFormat(reportFormat) + .withFilename(filename)); ////////////////////////////////////////////////////////////// // process the report's input fields, from the query string // @@ -239,7 +244,7 @@ public class QJavalinProcessHandler UnsafeFunction preAction = (PipedOutputStream pos) -> { - reportInput.setReportOutputStream(pos); + reportInput.getReportDestination().setReportOutputStream(pos); GenerateReportAction reportAction = new GenerateReportAction(); // any pre-action?? export uses this for "too many rows" checks... @@ -282,12 +287,24 @@ public class QJavalinProcessHandler // todo context.contentType(reportFormat.getMimeType()); context.header("Content-Disposition", "filename=" + context.pathParam("file")); - String filePath = context.queryParam("filePath"); - if(filePath == null) + String filePath = context.queryParam("filePath"); + String storageTableName = context.queryParam("storageTableName"); + String reference = context.queryParam("storageReference"); + + if(filePath != null) { - throw (new QBadRequestException("A filePath was not provided.")); + context.result(new FileInputStream(filePath)); } - context.result(new FileInputStream(filePath)); + else if(storageTableName != null && reference != null) + { + InputStream inputStream = new StorageAction().getInputStream(new StorageInput(storageTableName).withReference(reference)); + context.result(inputStream); + } + else + { + throw (new QBadRequestException("Missing query parameters to identify file to download")); + } + } catch(Exception e) { diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java index 76a879b0..9ae16b60 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java @@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import kong.unirest.HttpResponse; @@ -964,7 +965,7 @@ class QJavalinImplementationTest extends QJavalinTestBase Function makeNewInstanceWithBackendName = (backendName) -> { QInstance newInstance = new QInstance(); - newInstance.addBackend(new QBackendMetaData().withName(backendName).withBackendType("mock")); + newInstance.addBackend(new QBackendMetaData().withName(backendName).withBackendType(MockBackendModule.class)); if(!"invalid".equals(backendName)) { diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandlerTest.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandlerTest.java index cd25dee2..6a7ec840 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandlerTest.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandlerTest.java @@ -575,15 +575,15 @@ class QJavalinProcessHandlerTest extends QJavalinTestBase /******************************************************************************* - ** test calling download file with missing filePath + ** test calling download file without needed query-string params ** *******************************************************************************/ @Test - public void test_downloadFileMissingFilePath() + public void test_downloadFileMissingQueryStringParams() { HttpResponse response = Unirest.get(BASE_URL + "/download/myTestFile.txt").asString(); assertEquals(400, response.getStatus()); - assertTrue(response.getBody().contains("A filePath was not provided")); + assertTrue(response.getBody().contains("Missing query parameters to identify file")); } diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java index ae5d2694..54794851 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java @@ -74,6 +74,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.savedviews.SavedViewsMetaDataProvider; import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider; import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; @@ -226,7 +227,7 @@ public class TestUtils public static QBackendMetaData defineMemoryBackend() { return new QBackendMetaData() - .withBackendType("memory") + .withBackendType(MemoryBackendModule.class) .withName(BACKEND_NAME_MEMORY); } @@ -274,6 +275,7 @@ public class TestUtils } + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java index 620433ab..b0583cdb 100644 --- a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java +++ b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java @@ -60,6 +60,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; @@ -618,9 +619,10 @@ public class QPicoCliImplementation ///////////////////////////////////////////// ExportInput exportInput = new ExportInput(); exportInput.setTableName(tableName); - exportInput.setReportFormat(reportFormat); - exportInput.setFilename(filename); - exportInput.setReportOutputStream(outputStream); + exportInput.setReportDestination(new ReportDestination() + .withReportFormat(reportFormat) + .withFilename(filename) + .withReportOutputStream(outputStream)); exportInput.setLimit(subParseResult.matchedOptionValue("limit", null)); exportInput.setQueryFilter(generateQueryFilter(subParseResult)); diff --git a/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java b/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java index 97986500..850491cc 100644 --- a/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java +++ b/qqq-middleware-slack/src/main/java/com/kingsrook/qqq/slack/QSlackImplementation.java @@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; @@ -429,8 +430,11 @@ public class QSlackImplementation ExportInput exportInput = new ExportInput(); exportInput.setLimit(1000); exportInput.setTableName(tableName); - exportInput.setReportFormat(ReportFormat.valueOf(format)); - exportInput.setReportOutputStream(baos); + + exportInput.setReportDestination(new ReportDestination() + .withReportFormat(ReportFormat.valueOf(format)) + .withReportOutputStream(baos)); + setupSession(context, exportInput); ExportOutput output = new ExportAction().execute(exportInput);