Feedback from code reviews

src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java

@@ -35,7 +35,6 @@ import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
 import com.kingsrook.qqq.backend.core.state.StateType;
 import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
-import com.kingsrook.qqq.backend.core.utils.SleepUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -78,6 +77,11 @@ public class AsyncJobManager
             return (runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus));
          });
 
+         if(timeout == 0)
+         {
+            throw (new JobGoingAsyncException(uuidAndTypeStateKey.getUuid().toString()));
+         }
+
          T result = future.get(timeout, timeUnit);
          return (result);
       }
@@ -97,7 +101,7 @@ public class AsyncJobManager
    /*******************************************************************************
     ** Start a job, and always, just get back the job UUID.
     *******************************************************************************/
-   public <T extends Serializable> String startJob(AsyncJob<T> asyncJob)
+   public <T extends Serializable> String startJob(AsyncJob<T> asyncJob) throws QException
    {
       return (startJob("Anonymous", asyncJob));
    }
@@ -107,17 +111,17 @@ public class AsyncJobManager
    /*******************************************************************************
     ** Start a job, and always, just get back the job UUID.
     *******************************************************************************/
-   public <T extends Serializable> String startJob(String jobName, AsyncJob<T> asyncJob)
+   public <T extends Serializable> String startJob(String jobName, AsyncJob<T> asyncJob) throws QException
    {
-      UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.ASYNC_JOB_STATUS);
+      try
-      AsyncJobStatus      asyncJobStatus      = new AsyncJobStatus();
+      {
-      asyncJobStatus.setState(AsyncJobState.RUNNING);
+         startJob(jobName, 0, TimeUnit.MILLISECONDS, asyncJob);
-      getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
+         throw (new QException("Job was expected to go asynchronous, but did not"));
-
+      }
-      // todo - refactor, share
+      catch(JobGoingAsyncException jgae)
-      CompletableFuture.supplyAsync(() -> runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus));
+      {
-
+         return (jgae.getJobUUID());
-      return (uuidAndTypeStateKey.getUuid().toString());
+      }
    }
 
 
@@ -179,31 +183,4 @@ public class AsyncJobManager
       // return TempFileStateProvider.getInstance();
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public AsyncJobStatus waitForJob(String jobUUID) throws QException
-   {
-      AsyncJobState  asyncJobState  = AsyncJobState.RUNNING;
-      AsyncJobStatus asyncJobStatus = null;
-      while(asyncJobState.equals(AsyncJobState.RUNNING))
-      {
-         LOG.info("Sleeping, waiting on job [" + jobUUID + "]");
-         SleepUtils.sleep(100, TimeUnit.MILLISECONDS);
-
-         Optional<AsyncJobStatus> optionalAsyncJobStatus = getJobStatus(jobUUID);
-         if(optionalAsyncJobStatus.isEmpty())
-         {
-            // todo - ... maybe some version of try-again?
-            throw (new QException("Could not get status of report query job [" + jobUUID + "]"));
-         }
-         asyncJobStatus = optionalAsyncJobStatus.get();
-         asyncJobState = asyncJobStatus.getState();
-      }
-
-      return (asyncJobStatus);
-   }
-
 }

src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvReportStreamer.java

@@ -65,14 +65,13 @@ public class CsvReportStreamer implements ReportStreamerInterface
     **
     *******************************************************************************/
    @Override
-   public void start(ReportInput reportInput) throws QReportingException
+   public void start(ReportInput reportInput, List<QFieldMetaData> fields) throws QReportingException
    {
       this.reportInput = reportInput;
+      this.fields = fields;
       table = reportInput.getTable();
       outputStream = this.reportInput.getReportOutputStream();
 
-      fields = setupFieldList(table, reportInput);
-
       writeReportHeaderRow();
    }
 

src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelReportStreamer.java

@@ -72,17 +72,16 @@ public class ExcelReportStreamer implements ReportStreamerInterface
     **
     *******************************************************************************/
    @Override
-   public void start(ReportInput reportInput) throws QReportingException
+   public void start(ReportInput reportInput, List<QFieldMetaData> fields) throws QReportingException
    {
       this.reportInput = reportInput;
+      this.fields = fields;
       table = reportInput.getTable();
       outputStream = this.reportInput.getReportOutputStream();
 
       workbook = new Workbook(outputStream, "QQQ", null);
       worksheet = workbook.newWorksheet("Sheet 1");
 
-      fields = setupFieldList(table, reportInput);
-
       writeReportHeaderRow();
    }
 

src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java

@@ -22,10 +22,9 @@
 package com.kingsrook.qqq.backend.core.actions.reporting;
 
 
-import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Queue;
+import java.util.concurrent.ArrayBlockingQueue;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
@@ -35,16 +34,16 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  *******************************************************************************/
 public class RecordPipe
 {
-   private Queue<QRecord> queue = new ArrayDeque<>();
+   private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(10_000);
-
 
 
    /*******************************************************************************
     ** Add a record to the pipe
+    ** Returns true iff the record fit in the pipe; false if the pipe is currently full.
     *******************************************************************************/
-   public void addRecord(QRecord record)
+   public boolean addRecord(QRecord record)
    {
-      queue.add(record);
+      return (queue.offer(record));
    }
 
 

src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportAction.java

@@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@@ -69,6 +70,10 @@ public class ReportAction
    private boolean preExecuteRan       = false;
    private Integer countFromPreExecute = null;
 
+   private static final int TIMEOUT_AFTER_NO_RECORDS_MS = 10 * 60 * 1000;
+   private static final int MAX_SLEEP_MS                = 1000;
+   private static final int INIT_SLEEP_MS               = 10;
+
 
 
    /*******************************************************************************
@@ -137,7 +142,6 @@ public class ReportAction
 
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
       QBackendModuleInterface  backendModule            = qBackendModuleDispatcher.getQBackendModule(reportInput.getBackend());
-      QTableMetaData           table                    = reportInput.getTable();
 
       //////////////////////////
       // set up a query input //
@@ -160,64 +164,73 @@ public class ReportAction
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       ReportFormat            reportFormat   = reportInput.getReportFormat();
       ReportStreamerInterface reportStreamer = reportFormat.newReportStreamer();
-      reportStreamer.start(reportInput);
+      reportStreamer.start(reportInput, getFields(reportInput));
 
       //////////////////////////////////////////
       // run the query action as an async job //
       //////////////////////////////////////////
       AsyncJobManager asyncJobManager = new AsyncJobManager();
-      String          jobUUID         = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> (queryInterface.execute(queryInput)));
+      String          queryJobUUID    = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> (queryInterface.execute(queryInput)));
-      LOG.info("Started query job [" + jobUUID + "] for report");
+      LOG.info("Started query job [" + queryJobUUID + "] for report");
 
-      AsyncJobState  asyncJobState   = AsyncJobState.RUNNING;
+      AsyncJobState  queryJobState  = AsyncJobState.RUNNING;
       AsyncJobStatus asyncJobStatus = null;
-      int            nextSleepMillis = 10;
-      long           recordCount     = 0;
-      while(asyncJobState.equals(AsyncJobState.RUNNING))
-      {
-         int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe);
-         recordCount += recordsConsumed;
 
-         if(countFromPreExecute != null)
+      long recordCount           = 0;
+      int  nextSleepMillis       = INIT_SLEEP_MS;
+      long lastReceivedRecordsAt = System.currentTimeMillis();
+      long reportStartTime       = System.currentTimeMillis();
+
+      while(queryJobState.equals(AsyncJobState.RUNNING))
       {
-            LOG.info(String.format("Processed %,d of %,d records so far", recordCount, countFromPreExecute));
+         if(recordPipe.countAvailableRecords() == 0)
+         {
+            ///////////////////////////////////////////////////////////
+            // if the pipe is empty, sleep to let the producer work. //
+            // todo - smarter sleep?  like get notified vs. sleep?   //
+            ///////////////////////////////////////////////////////////
+            LOG.info("No records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work");
+            SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS);
+            nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS);
+
+            long timeSinceLastReceivedRecord = System.currentTimeMillis() - lastReceivedRecordsAt;
+            if(timeSinceLastReceivedRecord > TIMEOUT_AFTER_NO_RECORDS_MS)
+            {
+               throw (new QReportingException("Query action appears to have stopped producing records (last record received " + timeSinceLastReceivedRecord + " ms ago)."));
+            }
          }
          else
          {
-            LOG.info(String.format("Processed %,d records so far", recordCount));
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if the pipe has records, consume them.  reset the sleep timer so if we sleep again it'll be short. //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////
+            lastReceivedRecordsAt = System.currentTimeMillis();
+            nextSleepMillis = INIT_SLEEP_MS;
+
+            int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe);
+            recordCount += recordsConsumed;
+
+            LOG.info(countFromPreExecute != null
+               ? String.format("Processed %,d of %,d records so far", recordCount, countFromPreExecute)
+               : String.format("Processed %,d records so far", recordCount));
          }
 
-         if(recordsConsumed == 0)
+         ////////////////////////////////////
-         {
+         // refresh the query job's status //
-            ////////////////////////////////////////////////////////////////////////////
+         ////////////////////////////////////
-            // do we need to sleep to let the producer work?                          //
+         Optional<AsyncJobStatus> optionalAsyncJobStatus = asyncJobManager.getJobStatus(queryJobUUID);
-            // todo - smarter sleep?  like get notified vs. sleep? eventually a fail? //
-            ////////////////////////////////////////////////////////////////////////////
-            LOG.info("Read 0 records from pipe, sleeping to give producer a chance to work");
-            SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS);
-            while(recordPipe.countAvailableRecords() == 0)
-            {
-               nextSleepMillis = Math.min(nextSleepMillis * 2, 1000);
-               LOG.info("Still no records in the pipe, so sleeping more [" + nextSleepMillis + "]ms");
-               SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS);
-            }
-         }
-
-         nextSleepMillis = 10;
-
-         Optional<AsyncJobStatus> optionalAsyncJobStatus = asyncJobManager.getJobStatus(jobUUID);
          if(optionalAsyncJobStatus.isEmpty())
          {
             /////////////////////////////////////////////////
             // todo - ... maybe some version of try-again? //
             /////////////////////////////////////////////////
-            throw (new QException("Could not get status of report query job [" + jobUUID + "]"));
+            throw (new QException("Could not get status of report query job [" + queryJobUUID + "]"));
          }
          asyncJobStatus = optionalAsyncJobStatus.get();
-         asyncJobState = asyncJobStatus.getState();
+         queryJobState = asyncJobStatus.getState();
       }
 
-      LOG.info("Query job [" + jobUUID + "] for report completed with status: " + asyncJobStatus);
+      LOG.info("Query job [" + queryJobUUID + "] for report completed with status: " + asyncJobStatus);
 
       ///////////////////////////////////////////////////
       // send the final records to the report streamer //
@@ -225,6 +238,12 @@ public class ReportAction
       int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe);
       recordCount += recordsConsumed;
 
+      long reportEndTime = System.currentTimeMillis();
+      LOG.info((countFromPreExecute != null
+         ? String.format("Processed %,d of %,d records", recordCount, countFromPreExecute)
+         : String.format("Processed %,d records", recordCount))
+         + String.format(" at end of report in %,d ms (%.2f records/second).", (reportEndTime - reportStartTime), 1000d * (recordCount / (.001d + (reportEndTime - reportStartTime)))));
+
       //////////////////////////////////////////////////////////////////
       // Critical:  we must close the stream here as our final action //
       //////////////////////////////////////////////////////////////////
@@ -247,11 +266,40 @@ public class ReportAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<QFieldMetaData> getFields(ReportInput reportInput)
+   {
+      QTableMetaData table = reportInput.getTable();
+      if(reportInput.getFieldNames() != null)
+      {
+         return (reportInput.getFieldNames().stream().map(table::getField).toList());
+      }
+      else
+      {
+         return (new ArrayList<>(table.getFields().values()));
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    private void verifyCountUnderMax(ReportInput reportInput, QBackendModuleInterface backendModule, ReportFormat reportFormat) throws QException
    {
+      if(reportFormat.getMaxCols() != null)
+      {
+         List<QFieldMetaData> fields = getFields(reportInput);
+         if (fields.size() > reportFormat.getMaxCols())
+         {
+            throw (new QUserFacingException("The requested report would include more columns ("
+               + String.format("%,d", fields.size()) + ") than the maximum allowed ("
+               + String.format("%,d", reportFormat.getMaxCols()) + ") for the selected file format (" + reportFormat + ")."));
+         }
+      }
+
       if(reportFormat.getMaxRows() != null)
       {
          if(reportInput.getLimit() == null || reportInput.getLimit() > reportFormat.getMaxRows())

src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportStreamerInterface.java

@@ -22,12 +22,10 @@
 package com.kingsrook.qqq.backend.core.actions.reporting;
 
 
-import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 
 
 /*******************************************************************************
@@ -38,7 +36,7 @@ public interface ReportStreamerInterface
    /*******************************************************************************
     ** Called once, before any rows are available.  Meant to write a header, for example.
     *******************************************************************************/
-   void start(ReportInput reportInput) throws QReportingException;
+   void start(ReportInput reportInput, List<QFieldMetaData> fields) throws QReportingException;
 
    /*******************************************************************************
     ** Called as records flow into the pipe.
@@ -50,20 +48,4 @@ public interface ReportStreamerInterface
     *******************************************************************************/
    void finish() throws QReportingException;
 
-   /*******************************************************************************
-    ** (Ideally, protected) method used within report streamer implementations, to
-    ** map field names from reportInput into list of fieldMetaData.
-    *******************************************************************************/
-   default List<QFieldMetaData> setupFieldList(QTableMetaData table, ReportInput reportInput)
-   {
-      if(reportInput.getFieldNames() != null)
-      {
-         return (reportInput.getFieldNames().stream().map(table::getField).toList());
-      }
-      else
-      {
-         return (new ArrayList<>(table.getFields().values()));
-      }
-   }
-
 }

src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java

@@ -77,6 +77,7 @@ public class DeleteAction
             LOG.info("0 primaryKeys found.  Returning with no-op");
             DeleteOutput deleteOutput = new DeleteOutput();
             deleteOutput.setRecordsWithErrors(new ArrayList<>());
+            deleteOutput.setDeletedRecordCount(0);
             return (deleteOutput);
          }
       }

src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -246,14 +246,12 @@ public class QInstanceEnricher
          .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withIsRequired(true))
          .withComponent(new QFrontendComponentMetaData()
             .withType(QComponentType.HELP_TEXT)
-            .withValue("text", "Upload a CSV or XLSX file with the following columns: " + fieldsForHelpText));
+            // .withValue("text", "Upload a CSV or XLSX file with the following columns: " + fieldsForHelpText));
+            .withValue("text", "Upload a CSV file with the following columns: " + fieldsForHelpText));
 
       QBackendStepMetaData receiveFileStep = new QBackendStepMetaData()
          .withName("receiveFile")
          .withCode(new QCodeReference(BulkInsertReceiveFileStep.class))
-         .withInputData(new QFunctionInputMetaData()
-            // todo - our upload file as a field?  problem is, its type...
-            .withFieldList(List.of()))
          .withOutputMetaData(new QFunctionOutputMetaData()
             .withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER))));
 
@@ -268,9 +266,6 @@ public class QInstanceEnricher
       QBackendStepMetaData storeStep = new QBackendStepMetaData()
          .withName("storeRecords")
          .withCode(new QCodeReference(BulkInsertStoreRecordsStep.class))
-         .withInputData(new QFunctionInputMetaData()
-            // todo - our upload file as a field?  problem is, its type...
-            .withFieldList(List.of()))
          .withOutputMetaData(new QFunctionOutputMetaData()
             .withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER))));
 
@@ -342,15 +337,13 @@ public class QInstanceEnricher
       QBackendStepMetaData storeStep = new QBackendStepMetaData()
          .withName("storeRecords")
          .withCode(new QCodeReference(BulkEditStoreRecordsStep.class))
-         .withInputData(new QFunctionInputMetaData()
-            // todo - our upload file as a field?  problem is, its type...
-            .withFieldList(List.of()))
          .withOutputMetaData(new QFunctionOutputMetaData()
             .withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER))));
 
       QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData()
          .withName("results")
          .withRecordListFields(new ArrayList<>(table.getFields().values()))
+         .withViewField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, QFieldType.STRING))
          .withComponent(new QFrontendComponentMetaData()
             .withType(QComponentType.HELP_TEXT)
             .withValue("text", "The records below have been updated."));

src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/QUploadedFile.java

@@ -30,6 +30,8 @@ import java.io.Serializable;
  *******************************************************************************/
 public class QUploadedFile implements Serializable
 {
+   public static final String DEFAULT_UPLOADED_FILE_FIELD_NAME = "uploadedFileKey";
+
    private String filename;
    private byte[] bytes;
 

src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.processes;
 
 
 import java.io.Serializable;
+import java.math.BigDecimal;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
@@ -204,7 +205,7 @@ public class RunBackendStepOutput extends AbstractActionOutput
     *******************************************************************************/
    public String getValueString(String fieldName)
    {
-      return ((String) getValue(fieldName));
+      return (ValueUtils.getValueAsString(getValue(fieldName)));
    }
 
 
@@ -218,4 +219,26 @@ public class RunBackendStepOutput extends AbstractActionOutput
       return (ValueUtils.getValueAsInteger(getValue(fieldName)));
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for a single field's value
+    **
+    *******************************************************************************/
+   public Boolean getValueBoolean(String fieldName)
+   {
+      return (ValueUtils.getValueAsBoolean(getValue(fieldName)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for a single field's value
+    **
+    *******************************************************************************/
+   public BigDecimal getValueBigDecimal(String fieldName)
+   {
+      return (ValueUtils.getValueAsBigDecimal(getValue(fieldName)));
+   }
+
 }

src/main/java/com/kingsrook/qqq/backend/core/model/actions/reporting/ReportFormat.java

@@ -37,11 +37,12 @@ import org.dhatim.fastexcel.Worksheet;
  *******************************************************************************/
 public enum ReportFormat
 {
-   XLSX(Worksheet.MAX_ROWS, ExcelReportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
+   XLSX(Worksheet.MAX_ROWS, Worksheet.MAX_COLS, ExcelReportStreamer::new, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
-   CSV(null, CsvReportStreamer::new, "text/csv");
+   CSV(null, null, CsvReportStreamer::new, "text/csv");
 
 
    private final Integer maxRows;
+   private final Integer maxCols;
    private final String  mimeType;
 
    private final Supplier<? extends ReportStreamerInterface> streamerConstructor;
@@ -51,9 +52,10 @@ public enum ReportFormat
    /*******************************************************************************
     **
     *******************************************************************************/
-   ReportFormat(Integer maxRows, Supplier<? extends ReportStreamerInterface> streamerConstructor, String mimeType)
+   ReportFormat(Integer maxRows, Integer maxCols, Supplier<? extends ReportStreamerInterface> streamerConstructor, String mimeType)
    {
       this.maxRows = maxRows;
+      this.maxCols = maxCols;
       this.mimeType = mimeType;
       this.streamerConstructor = streamerConstructor;
    }
@@ -92,6 +94,16 @@ public enum ReportFormat
    }
 
 
+   /*******************************************************************************
+    ** Getter for maxCols
+    **
+    *******************************************************************************/
+   public Integer getMaxCols()
+   {
+      return maxCols;
+   }
+
+
 
    /*******************************************************************************
     ** Getter for mimeType

src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java

@@ -58,8 +58,16 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface
    @Override
    public void addRecord(QRecord record)
    {
-      recordPipe.addRecord(record);
+      if(!recordPipe.addRecord(record))
-      blockIfPipeIsTooFull();
+      {
+         do
+         {
+            LOG.debug("Record pipe.add failed (due to full pipe).  Blocking.");
+            SleepUtils.sleep(10, TimeUnit.MILLISECONDS);
+         }
+         while(!recordPipe.addRecord(record));
+         LOG.debug("Done blocking.");
+      }
    }
 
 

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java

@@ -42,6 +42,9 @@ public enum QFieldType
    HTML,
    PASSWORD,
    BLOB;
+   ///////////////////////////////////////////////////////////////////////
+   // keep these values in sync with QFieldType.ts in qqq-frontend-core //
+   ///////////////////////////////////////////////////////////////////////
 
 
 

src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java

@@ -24,7 +24,9 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
 
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -33,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -43,6 +46,9 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
  *******************************************************************************/
 public class BulkDeleteStoreStep implements BackendStep
 {
+   public static final String ERROR_COUNT = "errorCount";
+
+
 
    /*******************************************************************************
     **
@@ -50,7 +56,7 @@ public class BulkDeleteStoreStep implements BackendStep
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
-      runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting records in database...");
+      runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting records...");
       runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal();
 
       DeleteInput deleteInput = new DeleteInput(runBackendStepInput.getInstance());
@@ -78,12 +84,16 @@ public class BulkDeleteStoreStep implements BackendStep
             .toList();
          deleteInput.setPrimaryKeys(primaryKeyList);
       }
+      else
+      {
+         throw (new QException("Missing required inputs (queryFilterJSON or record list)"));
+      }
 
       DeleteAction deleteAction = new DeleteAction();
       DeleteOutput deleteOutput = deleteAction.execute(deleteInput);
 
-      // todo - something with the output!!
+      List<QRecord> recordsWithErrors = Objects.requireNonNullElse(deleteOutput.getRecordsWithErrors(), Collections.emptyList());
-      deleteOutput.getRecordsWithErrors();
+      runBackendStepOutput.addValue(ERROR_COUNT, recordsWithErrors.size());
 
       runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
    }

src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java

@@ -23,16 +23,11 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
 
 
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
@@ -68,26 +63,7 @@ public class BulkEditReceiveValuesStep implements BackendStep
          }
       }
 
-      /////////////////////////////////////////////////////////////////////
+      BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "will be");
-      // build the string to show the user what fields are being changed //
-      /////////////////////////////////////////////////////////////////////
-      List<String>   valuesBeingUpdated = new ArrayList<>();
-      QTableMetaData table              = runBackendStepInput.getTable();
-      for(String fieldName : enabledFields)
-      {
-         String       label = table.getField(fieldName).getLabel();
-         Serializable value = runBackendStepInput.getValue(fieldName);
-
-         if(StringUtils.hasContent(ValueUtils.getValueAsString(value)))
-         {
-            valuesBeingUpdated.add(label + " will be set to: " + value);
-         }
-         else
-         {
-            valuesBeingUpdated.add(label + " will be cleared out.");
-         }
-      }
-      runBackendStepOutput.addValue(FIELD_VALUES_BEING_UPDATED, String.join("\n", valuesBeingUpdated));
 
       runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
    }

src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java

@@ -69,7 +69,7 @@ public class BulkEditStoreRecordsStep implements BackendStep
          }
       }
 
-      runBackendStepInput.getAsyncJobCallback().updateStatus("Updating database...");
+      runBackendStepInput.getAsyncJobCallback().updateStatus("Storing updated records...");
       runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal();
 
       UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance());
@@ -83,6 +83,8 @@ public class BulkEditStoreRecordsStep implements BackendStep
       UpdateOutput updateOutput = updateAction.execute(updateInput);
 
       runBackendStepOutput.setRecords(updateOutput.getRecords());
+
+      BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "was");
    }
 
 }

src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java

@@ -0,0 +1,68 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Utility methods used for Bulk Edit steps
+ *******************************************************************************/
+public class BulkEditUtils
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void setFieldValuesBeingUpdated(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, String[] enabledFields, String verb)
+   {
+      /////////////////////////////////////////////////////////////////////
+      // build the string to show the user what fields are being changed //
+      /////////////////////////////////////////////////////////////////////
+      List<String>   valuesBeingUpdated = new ArrayList<>();
+      QTableMetaData table              = runBackendStepInput.getTable();
+      for(String fieldName : enabledFields)
+      {
+         String       label = table.getField(fieldName).getLabel();
+         Serializable value = runBackendStepInput.getValue(fieldName);
+
+         if(StringUtils.hasContent(ValueUtils.getValueAsString(value)))
+         {
+            valuesBeingUpdated.add(label + " " + verb + " set to: " + value);
+         }
+         else
+         {
+            valuesBeingUpdated.add(label + " " + verb + " cleared out");
+         }
+      }
+      runBackendStepOutput.addValue(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, String.join("\n", valuesBeingUpdated));
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java

@@ -23,10 +23,12 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
 
 
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
@@ -47,7 +49,7 @@ public class BulkInsertUtils
     *******************************************************************************/
    static List<QRecord> getQRecordsFromFile(RunBackendStepInput runBackendStepInput) throws QException
    {
-      AbstractStateKey        stateKey             = (AbstractStateKey) runBackendStepInput.getValue("uploadedFileKey");
+      AbstractStateKey        stateKey             = (AbstractStateKey) runBackendStepInput.getValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME);
       Optional<QUploadedFile> optionalUploadedFile = TempFileStateProvider.getInstance().get(QUploadedFile.class, stateKey);
       if(optionalUploadedFile.isEmpty())
       {
@@ -55,20 +57,28 @@ public class BulkInsertUtils
       }
 
       byte[] bytes    = optionalUploadedFile.get().getBytes();
+      String fileName = optionalUploadedFile.get().getFilename();
 
       /////////////////////////////////////////////////////
       // let the user specify field labels instead names //
       /////////////////////////////////////////////////////
       QTableMetaData        table     = runBackendStepInput.getTable();
+      String                tableName = runBackendStepInput.getTableName();
       QKeyBasedFieldMapping mapping   = new QKeyBasedFieldMapping();
       for(Map.Entry<String, QFieldMetaData> entry : table.getFields().entrySet())
       {
          mapping.addMapping(entry.getKey(), entry.getValue().getLabel());
       }
 
-      // todo - sniff out file type...
+      List<QRecord> qRecords;
-      String        tableName = runBackendStepInput.getTableName();
+      if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv"))
-      List<QRecord> qRecords  = new CsvToQRecordAdapter().buildRecordsFromCsv(new String(bytes), runBackendStepInput.getInstance().getTable(tableName), mapping);
+      {
+         qRecords = new CsvToQRecordAdapter().buildRecordsFromCsv(new String(bytes), runBackendStepInput.getInstance().getTable(tableName), mapping);
+      }
+      else
+      {
+         throw (new QUserFacingException("Unsupported file type."));
+      }
 
       ////////////////////////////////////////////////
       // remove values from any non-editable fields //

src/main/java/com/kingsrook/qqq/backend/core/state/StateType.java

@@ -32,6 +32,5 @@ public enum StateType
 {
    PROCESS_STATUS,
    ASYNC_JOB_STATUS,
-   ASYNC_JOB_RESULT,
    UPLOADED_FILE
 }

src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java

@@ -34,7 +34,9 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.apache.commons.io.FileUtils;
@@ -70,6 +72,27 @@ class ReportActionTest
 
 
 
+   /*******************************************************************************
+    ** This test runs for more records, to stress more of the pipe-filling and
+    ** other bits of the ReportAction.
+    *******************************************************************************/
+   @Test
+   public void testBigger() throws Exception
+   {
+      // int    recordCount = 2_000_000; // to really stress locally, use this.
+      int    recordCount = 200_000;
+      String filename    = "/tmp/ReportActionTest.csv";
+
+      runReport(recordCount, filename, ReportFormat.CSV, false);
+
+      File         file      = new File(filename);
+      List<String> fileLines = FileUtils.readLines(file, StandardCharsets.UTF_8.name());
+      assertEquals(recordCount + 1, fileLines.size());
+      assertTrue(file.delete());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -142,12 +165,49 @@ class ReportActionTest
       ReportInput reportInput = new ReportInput(TestUtils.defineInstance(), TestUtils.getMockSession());
       reportInput.setTableName("person");
 
-      ////////////////////////////////////////////////////////////////
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // use xlsx, which has a max-rows limit, to verify that code. //
+      // use xlsx, which has a max-rows limit, to verify that code runs, but doesn't throw when there aren't too many rows //
-      ////////////////////////////////////////////////////////////////
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       reportInput.setReportFormat(ReportFormat.XLSX);
 
       new ReportAction().preExecute(reportInput);
+
+      ////////////////////////////////////////////////////////////////////////////
+      // nothing to assert - but if preExecute throws, then the test will fail. //
+      ////////////////////////////////////////////////////////////////////////////
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTooManyColumns() throws QException
+   {
+      QTableMetaData wideTable = new QTableMetaData()
+         .withName("wide")
+         .withBackendName(TestUtils.DEFAULT_BACKEND_NAME);
+      for(int i = 0; i < ReportFormat.XLSX.getMaxCols() + 1; i++)
+      {
+         wideTable.addField(new QFieldMetaData("field" + i, QFieldType.STRING));
+      }
+
+      QInstance qInstance = TestUtils.defineInstance();
+      qInstance.addTable(wideTable);
+
+      ReportInput reportInput = new ReportInput(qInstance, TestUtils.getMockSession());
+      reportInput.setTableName("wide");
+
+      ////////////////////////////////////////////////////////////////
+      // use xlsx, which has a max-cols limit, to verify that code. //
+      ////////////////////////////////////////////////////////////////
+      reportInput.setReportFormat(ReportFormat.XLSX);
+
+      assertThrows(QUserFacingException.class, () ->
+      {
+         new ReportAction().preExecute(reportInput);
+      });
    }
 
 }

src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java

@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
@@ -50,6 +51,7 @@ class BulkDeleteStoreStepTest
 
       RunBackendStepOutput stepOutput = new RunBackendStepOutput();
       new BulkDeleteStoreStep().run(stepInput, stepOutput);
+      assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT));
    }
 
 
@@ -66,6 +68,7 @@ class BulkDeleteStoreStepTest
 
       RunBackendStepOutput stepOutput = new RunBackendStepOutput();
       new BulkDeleteStoreStep().run(stepInput, stepOutput);
+      assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT));
    }
 
 }

src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
 
 import java.util.List;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@@ -63,7 +64,7 @@ class BulkInsertReceiveFileStepTest
       RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
       stepInput.setSession(TestUtils.getMockSession());
       stepInput.setTableName(TestUtils.defineTablePerson().getName());
-      stepInput.addValue("uploadedFileKey", key);
+      stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
 
       RunBackendStepOutput stepOutput = new RunBackendStepOutput();
       new BulkInsertReceiveFileStep().run(stepInput, stepOutput);
@@ -78,4 +79,36 @@ class BulkInsertReceiveFileStepTest
       assertEquals(2, stepOutput.getValueInteger("noOfFileRows"));
    }
 
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testBadFileType() throws QException
+   {
+      ////////////////////////////////////////////////////////////////
+      // create an uploaded file, similar to how an http server may //
+      ////////////////////////////////////////////////////////////////
+      QUploadedFile qUploadedFile = new QUploadedFile();
+      qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); // todo - this is NOT excel content...
+      qUploadedFile.setFilename("test.xslx");
+      UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
+      TempFileStateProvider.getInstance().put(key, qUploadedFile);
+
+      ////////////////////////////
+      // setup and run the step //
+      ////////////////////////////
+      RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
+      stepInput.setSession(TestUtils.getMockSession());
+      stepInput.setTableName(TestUtils.defineTablePerson().getName());
+      stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
+
+      RunBackendStepOutput stepOutput = new RunBackendStepOutput();
+
+      assertThrows(QUserFacingException.class, () ->
+      {
+         new BulkInsertReceiveFileStep().run(stepInput, stepOutput);
+      });
+   }
+
 }

src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java

@@ -64,7 +64,7 @@ class BulkInsertStoreRecordsStepTest
       RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
       stepInput.setSession(TestUtils.getMockSession());
       stepInput.setTableName(TestUtils.defineTablePerson().getName());
-      stepInput.addValue("uploadedFileKey", key);
+      stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
 
       RunBackendStepOutput stepOutput = new RunBackendStepOutput();
       new BulkInsertStoreRecordsStep().run(stepInput, stepOutput);

src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -335,7 +335,7 @@ public class TestUtils
    public static String getPersonCsvHeader()
    {
       return ("""
-         "id","createDate","modifyDate","firstName","lastName","birthDate","email"\r
+         "id","createDate","modifyDate","firstName","lastName","birthDate","email"
          """);
    }
 
@@ -347,7 +347,7 @@ public class TestUtils
    public static String getPersonCsvHeaderUsingLabels()
    {
       return ("""
-         "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"\r
+         "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"
          """);
    }
 
@@ -359,7 +359,7 @@ public class TestUtils
    public static String getPersonCsvRow1()
    {
       return ("""
-         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"\r
+         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"
          """);
    }
 
@@ -371,7 +371,7 @@ public class TestUtils
    public static String getPersonCsvRow2()
    {
       return ("""
-         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"\r
+         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"
          """);
    }