Make postRun methods take a subclass of backendInput/Output, that make it clear they don't have the full record list

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -65,6 +65,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -355,7 +357,7 @@ public class GenerateReportAction
       ////////////////////////////////////////////////
       if(transformStep != null)
       {
-         transformStep.postRun(transformStepInput, transformStepOutput);
+         transformStep.postRun(new BackendStepPostRunInput(transformStepInput), new BackendStepPostRunOutput(transformStepOutput));
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java

@@ -64,7 +64,7 @@ public abstract class AbstractLoadStep implements BackendStep
     ** Allow subclasses to do an action after the run is complete - after the last
     ** page of records is passed in.
     *******************************************************************************/
-   public void postRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   public void postRun(BackendStepPostRunInput runBackendStepInput, BackendStepPostRunOutput runBackendStepOutput) throws QException
    {
       ////////////////////////
       // noop in base class //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java

@@ -63,7 +63,7 @@ public abstract class AbstractTransformStep implements BackendStep, ProcessSumma
     ** Allow subclasses to do an action after the run is complete - after the last
     ** page of records is passed in.
     *******************************************************************************/
-   public void postRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   public void postRun(BackendStepPostRunInput runBackendStepInput, BackendStepPostRunOutput runBackendStepOutput) throws QException
    {
       ////////////////////////
       // noop in base class //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java

@@ -138,8 +138,20 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
             runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, transformStep.doGetProcessSummary(runBackendStepOutput, true));
          }
 
-         transformStep.postRun(runBackendStepInput, runBackendStepOutput);
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         loadStep.postRun(runBackendStepInput, runBackendStepOutput);
+         // use a subclass of runBackendStepOutput that makes it clear you can't use the recordList, as it's a "preview/subset" record list //
+         // this prevents bugs where you might think you have the full record list, but really don't                                        //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         BackendStepPostRunOutput postRunOutput = new BackendStepPostRunOutput(runBackendStepOutput);
+         BackendStepPostRunInput  postRunInput  = new BackendStepPostRunInput(runBackendStepInput);
+         transformStep.postRun(postRunInput, postRunOutput);
+         loadStep.postRun(postRunInput, postRunOutput);
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // explicitly copy values back into the runStepOutput from the post-run output                                     //
+         // this might not be needed, since they (presumably) share a processState object, but just in case that changes... //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         runBackendStepOutput.setValues(postRunOutput.getValues());
 
          if(recordCount > 0)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java

@@ -124,7 +124,9 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
       updateRecordsWithDisplayValuesAndPossibleValues(runBackendStepInput, previewRecordList);
       runBackendStepOutput.setRecords(previewRecordList);
 
-      transformStep.postRun(runBackendStepInput, runBackendStepOutput);
+      BackendStepPostRunOutput postRunOutput = new BackendStepPostRunOutput(runBackendStepOutput);
+      BackendStepPostRunInput  postRunInput  = new BackendStepPostRunInput(runBackendStepInput);
+      transformStep.postRun(postRunInput, postRunOutput);
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java

@@ -108,7 +108,9 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back
       //////////////////////////////////////////////////////
       runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY, transformStep.doGetProcessSummary(runBackendStepOutput, false));
 
-      transformStep.postRun(runBackendStepInput, runBackendStepOutput);
+      BackendStepPostRunOutput postRunOutput = new BackendStepPostRunOutput(runBackendStepOutput);
+      BackendStepPostRunInput  postRunInput  = new BackendStepPostRunInput(runBackendStepInput);
+      transformStep.postRun(postRunInput, postRunOutput);
    }
 
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java

@@ -53,6 +53,7 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -332,6 +333,39 @@ public class StreamedETLWithFrontendProcessTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testPostRun() throws QException
+   {
+      QInstance instance = QContext.getQInstance();
+
+      ////////////////////////////////////////////////////////
+      // define the process - an ELT from Shapes to Persons //
+      ////////////////////////////////////////////////////////
+      QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
+         TestUtils.TABLE_NAME_SHAPE,
+         TestUtils.TABLE_NAME_PERSON,
+         ExtractViaQueryStep.class,
+         NoopTransformStep.class,
+         TestLoadPostRunStep.class);
+      process.setName("test");
+      process.setTableName(TestUtils.TABLE_NAME_SHAPE);
+      instance.addProcess(process);
+
+      instance.getTable(TestUtils.TABLE_NAME_PERSON).setBackendName(TestUtils.MEMORY_BACKEND_NAME);
+      TestUtils.insertDefaultShapes(instance);
+
+      /////////////////////
+      // run the process //
+      /////////////////////
+      RunProcessOutput runProcessOutput = runProcess(instance, process, Collections.emptyMap(), new Callback(), RunProcessInput.FrontendStepBehavior.SKIP);
+      assertEquals(47, runProcessOutput.getValues().get("valueFromPostStep"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -540,6 +574,49 @@ public class StreamedETLWithFrontendProcessTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class TestLoadPostRunStep extends AbstractLoadStep
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+      {
+         ///////////////////////////////////
+         // just pass the records through //
+         ///////////////////////////////////
+         for(QRecord record : runBackendStepInput.getRecords())
+         {
+            runBackendStepOutput.addRecord(record);
+         }
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void postRun(BackendStepPostRunInput runBackendStepInput, BackendStepPostRunOutput runBackendStepOutput) throws QException
+      {
+         assertThatThrownBy(() -> runBackendStepInput.getRecords())
+            .isInstanceOf(IllegalStateException.class);
+         assertThatThrownBy(() -> runBackendStepOutput.getRecords())
+            .isInstanceOf(IllegalStateException.class);
+
+         assertThat(runBackendStepInput.getPreviewRecordList()).isNotEmpty();
+         assertThat(runBackendStepOutput.getPreviewRecordList()).isNotEmpty();
+
+         runBackendStepOutput.addValue("valueFromPostStep", 47);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/