Merge pull request #127 from Kingsrook/feature/CE-1405-zero-day-ledger-billing

Feature/ce 1405 zero day ledger billing

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java

@@ -24,10 +24,12 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 import java.io.Serializable;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -143,4 +145,19 @@ public interface RecordCustomizerUtilityInterface
       }
    }
 
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default Map<Serializable, QRecord> getOldRecordMap(List<QRecord> oldRecordList, UpdateInput updateInput)
+   {
+      Map<Serializable, QRecord> oldRecordMap = new HashMap<>();
+      for(QRecord qRecord : oldRecordList)
+      {
+         oldRecordMap.put(qRecord.getValue(updateInput.getTable().getPrimaryKeyField()), qRecord);
+      }
+
+      return (oldRecordMap);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer;
+import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -302,10 +303,19 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       JoinsContext joinsContext = null;
       if(dataSource != null)
       {
+         ///////////////////////////////////////////////////////////////////////////////////////
+         // count records, if applicable, from the data source - for populating into the      //
+         // countByDataSource map, as well as for checking if too many rows (e.g., for excel) //
+         ///////////////////////////////////////////////////////////////////////////////////////
+         countDataSourceRecords(reportInput, dataSource, reportFormat);
+
+         ///////////////////////////////////////////////////////////////////////////////////////////
+         // if there's a source table, set up a joins context, to use below for looking up fields //
+         ///////////////////////////////////////////////////////////////////////////////////////////
          if(StringUtils.hasContent(dataSource.getSourceTable()))
          {
-            joinsContext = new JoinsContext(QContext.getQInstance(), dataSource.getSourceTable(), cloneDataSourceQueryJoins(dataSource), dataSource.getQueryFilter() == null ? null : dataSource.getQueryFilter().clone());
-            countDataSourceRecords(reportInput, dataSource, reportFormat);
+            QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
+            joinsContext = new JoinsContext(QContext.getQInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), queryFilter);
          }
       }
 
@@ -329,6 +339,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
             field.setName(column.getName());
             if(StringUtils.hasContent(column.getLabel()))
             {
+
                field.setLabel(column.getLabel());
             }
             fields.add(field);
@@ -346,23 +357,33 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
     *******************************************************************************/
    private void countDataSourceRecords(ReportInput reportInput, QReportDataSource dataSource, ReportFormat reportFormat) throws QException
    {
-      QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
-      setInputValuesInQueryFilter(reportInput, queryFilter);
-
-      CountInput countInput = new CountInput();
-      countInput.setTableName(dataSource.getSourceTable());
-      countInput.setFilter(queryFilter);
-      countInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
-      CountOutput countOutput = new CountAction().execute(countInput);
-
-      if(countOutput.getCount() != null)
+      Integer count = null;
+      if(dataSource.getCustomRecordSource() != null)
       {
-         countByDataSource.put(dataSource.getName(), countOutput.getCount());
+         // todo - add `count` method to interface?
+      }
+      else if(StringUtils.hasContent(dataSource.getSourceTable()))
+      {
+         QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
+         setInputValuesInQueryFilter(reportInput, queryFilter);
 
-         if(reportFormat.getMaxRows() != null && countOutput.getCount() > reportFormat.getMaxRows())
+         CountInput countInput = new CountInput();
+         countInput.setTableName(dataSource.getSourceTable());
+         countInput.setFilter(queryFilter);
+         countInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
+         CountOutput countOutput = new CountAction().execute(countInput);
+
+         count = countOutput.getCount();
+      }
+
+      if(count != null)
+      {
+         countByDataSource.put(dataSource.getName(), count);
+
+         if(reportFormat.getMaxRows() != null && count > reportFormat.getMaxRows())
          {
             throw (new QUserFacingException("The requested report would include more rows ("
-               + String.format("%,d", countOutput.getCount()) + ") than the maximum allowed ("
+               + String.format("%,d", count) + ") than the maximum allowed ("
                + String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ")."));
          }
       }
@@ -423,13 +444,19 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       String        tableLabel    = ObjectUtils.tryElse(() -> QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel(), Objects.requireNonNullElse(dataSource.getSourceTable(), ""));
       AtomicInteger consumedCount = new AtomicInteger(0);
 
-      /////////////////////////////////////////////////////////////////
-      // run a record pipe loop, over the query for this data source //
-      /////////////////////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      // run a record pipe loop, over the query (or other data-supplier/source) for this data source //
+      /////////////////////////////////////////////////////////////////////////////////////////////////
       RecordPipe recordPipe = new BufferedRecordPipe(1000);
       new AsyncRecordPipeLoop().run("Report[" + reportInput.getReportName() + "]", null, recordPipe, (callback) ->
       {
-         if(dataSource.getSourceTable() != null)
+         if(dataSource.getCustomRecordSource() != null)
+         {
+            ReportCustomRecordSourceInterface recordSource = QCodeLoader.getAdHoc(ReportCustomRecordSourceInterface.class, dataSource.getCustomRecordSource());
+            recordSource.execute(reportInput, dataSource, recordPipe);
+            return (true);
+         }
+         else if(dataSource.getSourceTable() != null)
          {
             QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
             setInputValuesInQueryFilter(reportInput, queryFilter);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/customizers/ReportCustomRecordSourceInterface.java

@@ -0,0 +1,43 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.customizers;
+
+
+import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
+
+
+/*******************************************************************************
+ ** Interface to be implemented to do a custom source of data for a report
+ ** (instead of just a query against a table).
+ *******************************************************************************/
+public interface ReportCustomRecordSourceInterface
+{
+
+   /***************************************************************************
+    ** Given the report input, put records into the pipe, for the report.
+    ***************************************************************************/
+   void execute(ReportInput reportInput, QReportDataSource reportDataSource, RecordPipe recordPipe) throws QException;
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java

@@ -124,10 +124,11 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
    private Writer              activeSheetWriter = null;
    private StreamedSheetWriter sheetWriter       = null;
 
-   private QReportView                       currentView      = null;
-   private Map<String, List<QFieldMetaData>> fieldsPerView    = new HashMap<>();
-   private Map<String, Integer>              rowsPerView      = new HashMap<>();
-   private Map<String, String>               labelViewsByName = new HashMap<>();
+   private QReportView                       currentView              = null;
+   private Map<String, List<QFieldMetaData>> fieldsPerView            = new HashMap<>();
+   private Map<String, Integer>              rowsPerView              = new HashMap<>();
+   private Map<String, String>               labelViewsByName         = new HashMap<>();
+   private Map<String, String>               sheetReferenceByViewName = new HashMap<>();
 
 
 
@@ -180,6 +181,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
             String    sheetReference = sheet.getPackagePart().getPartName().getName().substring(1);
             sheetMapByExcelReference.put(sheetReference, sheet);
             sheetMapByViewName.put(view.getName(), sheet);
+            sheetReferenceByViewName.put(view.getName(), sheetReference);
             sheetCounter++;
          }
 
@@ -446,7 +448,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
          // - with a new output stream writer    //
          // - and with a SpreadsheetWriter       //
          //////////////////////////////////////////
-         zipOutputStream.putNextEntry(new ZipEntry("xl/worksheets/sheet" + this.sheetIndex++ + ".xml"));
+         zipOutputStream.putNextEntry(new ZipEntry(sheetReferenceByViewName.get(view.getName())));
          activeSheetWriter = new OutputStreamWriter(zipOutputStream);
          sheetWriter = new StreamedSheetWriter(activeSheetWriter);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -50,6 +51,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperat
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@@ -64,6 +66,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
@@ -101,6 +104,8 @@ public class QueryAction
          throw (new QException("A table named [" + queryInput.getTableName() + "] was not found in the active QInstance"));
       }
 
+      validateFieldNamesToInclude(queryInput);
+
       QBackendMetaData backend = queryInput.getBackend();
       postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole());
       this.queryInput = queryInput;
@@ -158,6 +163,109 @@ public class QueryAction
 
 
 
+   /***************************************************************************
+    ** if QueryInput contains a set of FieldNamesToInclude, then validate that
+    ** those are known field names in the table being queried, or a selected
+    ** queryJoin.
+    ***************************************************************************/
+   static void validateFieldNamesToInclude(QueryInput queryInput) throws QException
+   {
+      Set<String> fieldNamesToInclude = queryInput.getFieldNamesToInclude();
+      if(fieldNamesToInclude == null)
+      {
+         ////////////////////////////////
+         // null set means select all. //
+         ////////////////////////////////
+         return;
+      }
+
+      if(fieldNamesToInclude.isEmpty())
+      {
+         /////////////////////////////////////
+         // empty set, however, is an error //
+         /////////////////////////////////////
+         throw (new QException("An empty set of fieldNamesToInclude was given as queryInput, which is not allowed."));
+      }
+
+      List<String>                unrecognizedFieldNames = new ArrayList<>();
+      Map<String, QTableMetaData> selectedQueryJoins     = null;
+      for(String fieldName : fieldNamesToInclude)
+      {
+         if(fieldName.contains("."))
+         {
+            ////////////////////////////////////////////////
+            // handle names with dots - fields from joins //
+            ////////////////////////////////////////////////
+            String[] parts = fieldName.split("\\.");
+            if(parts.length != 2)
+            {
+               unrecognizedFieldNames.add(fieldName);
+            }
+            else
+            {
+               String tableOrAlias  = parts[0];
+               String fieldNamePart = parts[1];
+
+               ////////////////////////////////////////////
+               // build map of queryJoins being selected //
+               ////////////////////////////////////////////
+               if(selectedQueryJoins == null)
+               {
+                  selectedQueryJoins = new HashMap<>();
+                  for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryInput.getQueryJoins()))
+                  {
+                     if(queryJoin.getSelect())
+                     {
+                        String         joinTableOrAlias = queryJoin.getJoinTableOrItsAlias();
+                        QTableMetaData joinTable        = QContext.getQInstance().getTable(queryJoin.getJoinTable());
+                        if(joinTable != null)
+                        {
+                           selectedQueryJoins.put(joinTableOrAlias, joinTable);
+                        }
+                     }
+                  }
+               }
+
+               if(!selectedQueryJoins.containsKey(tableOrAlias))
+               {
+                  ///////////////////////////////////////////
+                  // unrecognized tableOrAlias is an error //
+                  ///////////////////////////////////////////
+                  unrecognizedFieldNames.add(fieldName);
+               }
+               else
+               {
+                  QTableMetaData joinTable = selectedQueryJoins.get(tableOrAlias);
+                  if(!joinTable.getFields().containsKey(fieldNamePart))
+                  {
+                     //////////////////////////////////////////////////////////
+                     // unrecognized field within the join table is an error //
+                     //////////////////////////////////////////////////////////
+                     unrecognizedFieldNames.add(fieldName);
+                  }
+               }
+            }
+         }
+         else
+         {
+            ///////////////////////////////////////////////////////////////////////
+            // non-join fields - just ensure field name is in table's fields map //
+            ///////////////////////////////////////////////////////////////////////
+            if(!queryInput.getTable().getFields().containsKey(fieldName))
+            {
+               unrecognizedFieldNames.add(fieldName);
+            }
+         }
+      }
+
+      if(!unrecognizedFieldNames.isEmpty())
+      {
+         throw (new QException("QueryInput contained " + unrecognizedFieldNames.size() + " unrecognized field name" + StringUtils.plural(unrecognizedFieldNames) + ": " + StringUtils.join(",", unrecognizedFieldNames)));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** shorthand way to call for the most common use-case, when you just want the
     ** records to be returned, and you just want to pass in a table name and filter.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface;
 import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
@@ -1660,9 +1661,12 @@ public class QInstanceValidator
 
                   String dataSourceErrorPrefix = "Report " + reportName + " data source " + dataSource.getName() + " ";
 
+                  boolean hasASource = false;
+
                   if(StringUtils.hasContent(dataSource.getSourceTable()))
                   {
-                     assertCondition(dataSource.getStaticDataSupplier() == null, dataSourceErrorPrefix + "has both a sourceTable and a staticDataSupplier (exactly 1 is required).");
+                     hasASource = true;
+                     assertCondition(dataSource.getStaticDataSupplier() == null, dataSourceErrorPrefix + "has both a sourceTable and a staticDataSupplier (not compatible together).");
                      if(assertCondition(qInstance.getTable(dataSource.getSourceTable()) != null, dataSourceErrorPrefix + "source table " + dataSource.getSourceTable() + " is not a table in this instance."))
                      {
                         if(dataSource.getQueryFilter() != null)
@@ -1671,14 +1675,21 @@ public class QInstanceValidator
                         }
                      }
                   }
-                  else if(dataSource.getStaticDataSupplier() != null)
+
+                  if(dataSource.getStaticDataSupplier() != null)
                   {
+                     assertCondition(dataSource.getCustomRecordSource() == null, dataSourceErrorPrefix + "has both a staticDataSupplier and a customRecordSource (not compatible together).");
+                     hasASource = true;
                      validateSimpleCodeReference(dataSourceErrorPrefix, dataSource.getStaticDataSupplier(), Supplier.class);
                   }
-                  else
+
+                  if(dataSource.getCustomRecordSource() != null)
                   {
-                     errors.add(dataSourceErrorPrefix + "does not have a sourceTable or a staticDataSupplier (exactly 1 is required).");
+                     hasASource = true;
+                     validateSimpleCodeReference(dataSourceErrorPrefix, dataSource.getCustomRecordSource(), ReportCustomRecordSourceInterface.class);
                   }
+
+                  assertCondition(hasASource, dataSourceErrorPrefix + "does not have a sourceTable, customRecordSource, or a staticDataSupplier.");
                }
             }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java

@@ -66,6 +66,14 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
    private List<QueryJoin> queryJoins     = null;
    private boolean         selectDistinct = false;
 
+   /////////////////////////////////////////////////////////////////////////////
+   // if this set is null, then the default (all fields) should be included   //
+   // if it's an empty set, that should throw an error                        //
+   // or if there are any fields in it that aren't valid fields on the table, //
+   // or in a selected queryJoin.                                             //
+   /////////////////////////////////////////////////////////////////////////////
+   private Set<String> fieldNamesToInclude;
+
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // if you say you want to includeAssociations, you can limit which ones by passing them in associationNamesToInclude. //
    // if you leave it null, you get all associations defined on the table.  if you pass it as empty, you get none.       //
@@ -686,4 +694,35 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
       return (queryHints.contains(queryHint));
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for fieldNamesToInclude
+    *******************************************************************************/
+   public Set<String> getFieldNamesToInclude()
+   {
+      return (this.fieldNamesToInclude);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldNamesToInclude
+    *******************************************************************************/
+   public void setFieldNamesToInclude(Set<String> fieldNamesToInclude)
+   {
+      this.fieldNamesToInclude = fieldNamesToInclude;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldNamesToInclude
+    *******************************************************************************/
+   public QueryInput withFieldNamesToInclude(Set<String> fieldNamesToInclude)
+   {
+      this.fieldNamesToInclude = fieldNamesToInclude;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportDataSource.java

@@ -32,6 +32,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 /*******************************************************************************
  ** Meta-data definition of a source of data for a report (e.g., a table and query
  ** filter or custom-code reference).
+ **
+ ** Runs in 3 modes:
+ **
+ ** - If a customRecordSource is specified, then that code is executed to get the records.
+ ** - else, if a sourceTable is specified, then the corresponding queryFilter
+ ** (optionally along with queryJoins and queryInputCustomizer) is used.
+ ** - else a staticDataSupplier is used.
  *******************************************************************************/
 public class QReportDataSource
 {
@@ -44,6 +51,7 @@ public class QReportDataSource
 
    private QCodeReference queryInputCustomizer;
    private QCodeReference staticDataSupplier;
+   private QCodeReference customRecordSource;
 
 
 
@@ -265,4 +273,35 @@ public class QReportDataSource
       return (this);
    }
 
+
+   /*******************************************************************************
+    ** Getter for customRecordSource
+    *******************************************************************************/
+   public QCodeReference getCustomRecordSource()
+   {
+      return (this.customRecordSource);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for customRecordSource
+    *******************************************************************************/
+   public void setCustomRecordSource(QCodeReference customRecordSource)
+   {
+      this.customRecordSource = customRecordSource;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for customRecordSource
+    *******************************************************************************/
+   public QReportDataSource withCustomRecordSource(QCodeReference customRecordSource)
+   {
+      this.customRecordSource = customRecordSource;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcess.java

@@ -46,7 +46,8 @@ public class BasicRunReportProcess
    public static final String STEP_NAME_EXECUTE = "execute";
    public static final String STEP_NAME_ACCESS  = "accessReport";
 
-   public static final String FIELD_REPORT_NAME = "reportName";
+   public static final String FIELD_REPORT_NAME   = "reportName";
+   public static final String FIELD_REPORT_FORMAT = "reportFormat";
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/ExecuteReportStep.java

@@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
@@ -50,6 +51,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 public class ExecuteReportStep implements BackendStep
 {
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -58,9 +60,10 @@ public class ExecuteReportStep implements BackendStep
    {
       try
       {
-         String          reportName = runBackendStepInput.getValueString("reportName");
-         QReportMetaData report     = QContext.getQInstance().getReport(reportName);
-         File            tmpFile    = File.createTempFile(reportName, ".xlsx", new File("/tmp/"));
+         ReportFormat    reportFormat = getReportFormat(runBackendStepInput);
+         String          reportName   = runBackendStepInput.getValueString("reportName");
+         QReportMetaData report       = QContext.getQInstance().getReport(reportName);
+         File            tmpFile      = File.createTempFile(reportName, "." + reportFormat.getExtension());
 
          runBackendStepInput.getAsyncJobCallback().updateStatus("Generating Report");
 
@@ -69,7 +72,7 @@ public class ExecuteReportStep implements BackendStep
             ReportInput reportInput = new ReportInput();
             reportInput.setReportName(reportName);
             reportInput.setReportDestination(new ReportDestination()
-               .withReportFormat(ReportFormat.XLSX) // todo - variable
+               .withReportFormat(reportFormat)
                .withReportOutputStream(reportOutputStream));
 
             Map<String, Serializable> values = runBackendStepInput.getValues();
@@ -79,7 +82,7 @@ public class ExecuteReportStep implements BackendStep
 
             String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, report);
 
-            runBackendStepOutput.addValue("downloadFileName", downloadFileBaseName + ".xlsx");
+            runBackendStepOutput.addValue("downloadFileName", downloadFileBaseName + "." + reportFormat.getExtension());
             runBackendStepOutput.addValue("serverFilePath", tmpFile.getCanonicalPath());
          }
       }
@@ -91,6 +94,22 @@ public class ExecuteReportStep implements BackendStep
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private ReportFormat getReportFormat(RunBackendStepInput runBackendStepInput) throws QUserFacingException
+   {
+      String reportFormatInput = runBackendStepInput.getValueString(BasicRunReportProcess.FIELD_REPORT_FORMAT);
+      if(StringUtils.hasContent(reportFormatInput))
+      {
+         return (ReportFormat.fromString(reportFormatInput));
+      }
+
+      return (ReportFormat.XLSX);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java

@@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.BaseTest;
@@ -38,6 +40,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -54,6 +57,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -541,4 +545,89 @@ class QueryActionTest extends BaseTest
       assertEquals(1, QueryAction.execute(TestUtils.TABLE_NAME_SHAPE, new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "SqUaRe"))).size());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testValidateFieldNamesToInclude() throws QException
+   {
+      /////////////////////////////
+      // cases that do not throw //
+      /////////////////////////////
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(null));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("id")));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("id", "firstName", "lastName", "birthDate", "modifyDate", "createDate")));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
+         .withFieldNamesToInclude(Set.of("id", "firstName", "lastName", "birthDate", "modifyDate", "createDate")));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
+         .withFieldNamesToInclude(Set.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", TestUtils.TABLE_NAME_SHAPE + ".noOfSides")));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true).withAlias("s"))
+         .withFieldNamesToInclude(Set.of("id", "s.id", "s.noOfSides")));
+
+      //////////////////////////
+      // cases that do throw! //
+      //////////////////////////
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(new HashSet<>())))
+         .hasMessageContaining("empty set of fieldNamesToInclude");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("notAField"))))
+         .hasMessageContaining("1 unrecognized field name: notAField");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("notAField", "alsoNot")))))
+         .hasMessageContaining("2 unrecognized field names: notAField,alsoNot");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("notAField", "alsoNot", "join.not")))))
+         .hasMessageContaining("3 unrecognized field names: notAField,alsoNot,join.not");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE)) // oops, didn't select it!
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", TestUtils.TABLE_NAME_SHAPE + ".noOfSides")))))
+         .hasMessageContaining("2 unrecognized field names: shape.id,shape.noOfSides");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
+         .withFieldNamesToInclude(Set.of(TestUtils.TABLE_NAME_SHAPE + ".noField"))))
+         .hasMessageContaining("1 unrecognized field name: shape.noField");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true).withAlias("s")) // oops, not using alias
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", "s.noOfSides")))))
+         .hasMessageContaining("1 unrecognized field name: shape.id");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", "noOfSides")))))
+         .hasMessageContaining("1 unrecognized field name: noOfSides");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin("noJoinTable").withSelect(true))
+         .withFieldNamesToInclude(Set.of("noJoinTable.id"))))
+         .hasMessageContaining("1 unrecognized field name: noJoinTable.id");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("noJoin.id"))))
+         .hasMessageContaining("1 unrecognized field name: noJoin.id");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("noDb.noJoin.id"))))
+         .hasMessageContaining("1 unrecognized field name: noDb.noJoin.id");
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.instances;
 
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -38,6 +39,8 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarCh
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ParentWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.processes.CancelProcessActionTest;
+import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
@@ -45,6 +48,7 @@ import com.kingsrook.qqq.backend.core.instances.validation.plugins.AlwaysFailsPr
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -1712,7 +1716,7 @@ public class QInstanceValidatorTest extends BaseTest
    @Test
    void testReportDataSourceStaticDataSupplier()
    {
-      assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0).withStaticDataSupplier(new QCodeReference()),
+      assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0).withStaticDataSupplier(new QCodeReference(TestReportStaticDataSupplier.class)),
          "has both a sourceTable and a staticDataSupplier");
 
       assertValidationFailureReasons((qInstance) ->
@@ -1720,16 +1724,43 @@ public class QInstanceValidatorTest extends BaseTest
          QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
          dataSource.setSourceTable(null);
          dataSource.setStaticDataSupplier(new QCodeReference(null, QCodeType.JAVA));
-      },
-         "missing a code reference name");
+      }, "missing a code reference name");
 
       assertValidationFailureReasons((qInstance) ->
       {
          QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
          dataSource.setSourceTable(null);
          dataSource.setStaticDataSupplier(new QCodeReference(ArrayList.class));
-      },
-         "is not of the expected type");
+      }, "is not of the expected type");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testReportDataSourceCustomRecordSource()
+   {
+      assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0)
+            .withSourceTable(null)
+            .withStaticDataSupplier(new QCodeReference(TestReportStaticDataSupplier.class))
+            .withCustomRecordSource(new QCodeReference(TestReportCustomRecordSource.class)),
+         "has both a staticDataSupplier and a customRecordSource");
+
+      assertValidationFailureReasons((qInstance) ->
+      {
+         QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
+         dataSource.setSourceTable(null);
+         dataSource.setCustomRecordSource(new QCodeReference(null, QCodeType.JAVA));
+      }, "missing a code reference name");
+
+      assertValidationFailureReasons((qInstance) ->
+      {
+         QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
+         dataSource.setSourceTable(null);
+         dataSource.setCustomRecordSource(new QCodeReference(ArrayList.class));
+      }, "is not of the expected type");
    }
 
 
@@ -2371,5 +2402,38 @@ public class QInstanceValidatorTest extends BaseTest
     *******************************************************************************/
    public static class ValidAuthCustomizer implements QAuthenticationModuleCustomizerInterface {}
 
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static class TestReportStaticDataSupplier implements Supplier<List<List<Serializable>>>
+   {
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public List<List<Serializable>> get()
+      {
+         return List.of();
+      }
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static class TestReportCustomRecordSource implements ReportCustomRecordSourceInterface
+   {
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public void execute(ReportInput reportInput, QReportDataSource reportDataSource, RecordPipe recordPipe) throws QException
+      {
+
+      }
+   }
 }
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcessTest.java

@@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
@@ -43,6 +44,7 @@ import static org.assertj.core.api.Assertions.assertThat;
  *******************************************************************************/
 class BasicRunReportProcessTest extends BaseTest
 {
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -74,6 +76,18 @@ class BasicRunReportProcessTest extends BaseTest
       runProcessOutput = new RunProcessAction().execute(runProcessInput);
       assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo(BasicRunReportProcess.STEP_NAME_ACCESS);
       assertThat(runProcessOutput.getValues()).containsKeys("downloadFileName", "serverFilePath");
+
+      ///////////////////////////////////
+      // assert we get xlsx by default //
+      ///////////////////////////////////
+      assertThat(runProcessOutput.getValueString("downloadFileName")).endsWith(".xlsx");
+
+      /////////////////////////////////////////////////////
+      // re-run, requesting CSV, then assert we get that //
+      /////////////////////////////////////////////////////
+      runProcessInput.addValue(BasicRunReportProcess.FIELD_REPORT_FORMAT, ReportFormat.CSV.name());
+      runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      assertThat(runProcessOutput.getValueString("downloadFileName")).endsWith(".csv");
    }
 
 }