From 25c9376ce474ba4cbaab65b48aa4282bc3f73562 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 6 Sep 2022 15:15:04 -0500 Subject: [PATCH] QQQ-37 update streamed-etl steps to not have to use different record-list --- .../processes/RunBackendStepInput.java | 24 ++++++ .../AbstractLoadStep.java | 58 ++------------ .../AbstractTransformStep.java | 58 ++------------ .../LoadViaInsertStep.java | 4 +- .../LoadViaUpdateStep.java | 4 +- .../StreamedBackendStepInput.java | 77 +++++++++++++++++++ .../StreamedBackendStepOutput.java | 74 ++++++++++++++++++ .../StreamedETLExecuteStep.java | 21 +++-- .../StreamedETLPreviewStep.java | 11 ++- .../StreamedETLValidateStep.java | 13 +++- .../StreamedETLWithFrontendProcessTest.java | 12 +-- .../clonepeople/ClonePeopleTransformStep.java | 4 +- .../ClonePeopleTransformStepTest.java | 2 +- 13 files changed, 229 insertions(+), 133 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepInput.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepOutput.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index cab801b9..69cf5787 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -51,6 +51,10 @@ public class RunBackendStepInput extends AbstractActionInput private QProcessCallback callback; private AsyncJobCallback asyncJobCallback; + //////////////////////////////////////////////////////////////////////////// + // note - new fields should generally be added in method: cloneFieldsInto // + //////////////////////////////////////////////////////////////////////////// + /******************************************************************************* @@ -85,6 +89,25 @@ public class RunBackendStepInput extends AbstractActionInput + /******************************************************************************* + ** Kinda like a reverse copy-constructor -- for a subclass that wants all the + ** field values from this object. Keep this in sync with the fields in this class! + ** + ** Of note - the processState does NOT get cloned - because... well, in our first + ** use-case (a subclass that doesn't WANT the same/full state), that's what we needed. + *******************************************************************************/ + public void cloneFieldsInto(RunBackendStepInput target) + { + target.setStepName(getStepName()); + target.setSession(getSession()); + target.setTableName(getTableName()); + target.setProcessName(getProcessName()); + target.setAsyncJobCallback(getAsyncJobCallback()); + target.setValues(getValues()); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -429,4 +452,5 @@ public class RunBackendStepInput extends AbstractActionInput } return (asyncJobCallback); } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java index 55b82768..58dcbc3a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java @@ -22,31 +22,27 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; -import java.util.ArrayList; -import java.util.List; import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; /******************************************************************************* ** Base class for the Load (aka, store) logic of Streamed ETL processes. ** - ** Records are to be read out of the inputRecordPage field, and after storing, - ** should be written to the outputRecordPage. That is to say, DO NOT use the - ** recordList in the step input/output objects. + ** Records are to be read out of the input object's Records field, and after storing, + ** should be written to the output object's Records, noting that when running + ** as a streamed-ETL process, those input & output objects will be instances of + ** the StreamedBackendStep{Input,Output} classes, that will be associated with + ** a page of records flowing thorugh a pipe. ** ** Also - use the transaction member variable!!! *******************************************************************************/ public abstract class AbstractLoadStep implements BackendStep { - private List inputRecordPage = new ArrayList<>(); - private List outputRecordPage = new ArrayList<>(); - private Optional transaction = Optional.empty(); @@ -87,50 +83,6 @@ public abstract class AbstractLoadStep implements BackendStep - /******************************************************************************* - ** Getter for recordPage - ** - *******************************************************************************/ - public List getInputRecordPage() - { - return inputRecordPage; - } - - - - /******************************************************************************* - ** Setter for recordPage - ** - *******************************************************************************/ - public void setInputRecordPage(List inputRecordPage) - { - this.inputRecordPage = inputRecordPage; - } - - - - /******************************************************************************* - ** Getter for outputRecordPage - ** - *******************************************************************************/ - public List getOutputRecordPage() - { - return outputRecordPage; - } - - - - /******************************************************************************* - ** Setter for outputRecordPage - ** - *******************************************************************************/ - public void setOutputRecordPage(List outputRecordPage) - { - this.outputRecordPage = outputRecordPage; - } - - - /******************************************************************************* ** Setter for transaction ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java index ffe4e7ef..200f895a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java @@ -22,27 +22,24 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; -import java.util.ArrayList; -import java.util.List; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; /******************************************************************************* ** Base class for the Transform logic of Streamed ETL processes. ** - ** Records are to be read out of the inputRecordPage field, and after transformation, - ** should be written to the outputRecordPage. That is to say, DO NOT use the - ** recordList in the step input/output objects. + ** Records are to be read out of the input object's Records field, and after storing, + ** should be written to the output object's Records, noting that when running + ** as a streamed-ETL process, those input & output objects will be instances of + ** the StreamedBackendStep{Input,Output} classes, that will be associated with + ** a page of records flowing through a pipe. + ** *******************************************************************************/ public abstract class AbstractTransformStep implements BackendStep, ProcessSummaryProviderInterface { - private List inputRecordPage = new ArrayList<>(); - private List outputRecordPage = new ArrayList<>(); - /******************************************************************************* @@ -69,47 +66,4 @@ public abstract class AbstractTransformStep implements BackendStep, ProcessSumma //////////////////////// } - - - /******************************************************************************* - ** Getter for recordPage - ** - *******************************************************************************/ - public List getInputRecordPage() - { - return inputRecordPage; - } - - - - /******************************************************************************* - ** Setter for recordPage - ** - *******************************************************************************/ - public void setInputRecordPage(List inputRecordPage) - { - this.inputRecordPage = inputRecordPage; - } - - - - /******************************************************************************* - ** Getter for outputRecordPage - ** - *******************************************************************************/ - public List getOutputRecordPage() - { - return outputRecordPage; - } - - - - /******************************************************************************* - ** Setter for outputRecordPage - ** - *******************************************************************************/ - public void setOutputRecordPage(List outputRecordPage) - { - this.outputRecordPage = outputRecordPage; - } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java index fe0f1a29..4903a710 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java @@ -52,10 +52,10 @@ public class LoadViaInsertStep extends AbstractLoadStep InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance()); insertInput.setSession(runBackendStepInput.getSession()); insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE)); - insertInput.setRecords(getInputRecordPage()); + insertInput.setRecords(runBackendStepInput.getRecords()); getTransaction().ifPresent(insertInput::setTransaction); InsertOutput insertOutput = new InsertAction().execute(insertInput); - getOutputRecordPage().addAll(insertOutput.getRecords()); + runBackendStepOutput.getRecords().addAll(insertOutput.getRecords()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java index 69eaffd1..39572ee1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java @@ -54,10 +54,10 @@ public class LoadViaUpdateStep extends AbstractLoadStep UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance()); updateInput.setSession(runBackendStepInput.getSession()); updateInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE)); - updateInput.setRecords(getInputRecordPage()); + updateInput.setRecords(runBackendStepInput.getRecords()); getTransaction().ifPresent(updateInput::setTransaction); UpdateOutput updateOutput = new UpdateAction().execute(updateInput); - getOutputRecordPage().addAll(updateOutput.getRecords()); + runBackendStepOutput.getRecords().addAll(updateOutput.getRecords()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepInput.java new file mode 100644 index 00000000..14b8f10e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepInput.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Subclass of RunBackendStepInput, meant for use in the pseudo-steps used by + ** the Streamed-ETL-with-frontend processes - where the Record list is not the + ** full process's record list - rather - is just a page at a time -- so this class + ** overrides the getRecords and setRecords method, to just work with that page. + ** + ** Note - of importance over time may be the RunBackendStepInput::cloneFieldsInto + ** method - e.g., if new fields are added to that class! + *******************************************************************************/ +public class StreamedBackendStepInput extends RunBackendStepInput +{ + private List inputRecords; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public StreamedBackendStepInput(RunBackendStepInput runBackendStepInput, List inputRecords) + { + super(runBackendStepInput.getInstance()); + runBackendStepInput.cloneFieldsInto(this); + this.inputRecords = inputRecords; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setRecords(List records) + { + this.inputRecords = records; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List getRecords() + { + return (inputRecords); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepOutput.java new file mode 100644 index 00000000..f750e67e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepOutput.java @@ -0,0 +1,74 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Subclass of RunBackendStepOutput, meant for use in the pseudo-steps used by + ** the Streamed-ETL-with-frontend processes - where the Record list is not the + ** full process's record list - rather - is just a page at a time -- so this class + ** overrides the getRecords and setRecords method, to just work with that page. + *******************************************************************************/ +public class StreamedBackendStepOutput extends RunBackendStepOutput +{ + private List outputRecords = new ArrayList<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public StreamedBackendStepOutput(RunBackendStepOutput runBackendStepOutput) + { + super(); + setValues(runBackendStepOutput.getValues()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setRecords(List records) + { + this.outputRecords = records; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List getRecords() + { + return (outputRecords); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java index 2c5f413f..4f33651f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java @@ -144,27 +144,32 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe /////////////////////////////////// List qRecords = recordPipe.consumeAvailableRecords(); + /////////////////////////////////////////////////////////////////////// + // make streamed input & output objects from the run input & outputs // + /////////////////////////////////////////////////////////////////////// + StreamedBackendStepInput streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, qRecords); + StreamedBackendStepOutput streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput); + ///////////////////////////////////////////////////// // pass the records through the transform function // ///////////////////////////////////////////////////// - transformStep.setInputRecordPage(qRecords); - transformStep.setOutputRecordPage(new ArrayList<>()); - transformStep.run(runBackendStepInput, runBackendStepOutput); + transformStep.run(streamedBackendStepInput, streamedBackendStepOutput); //////////////////////////////////////////////// // pass the records through the load function // //////////////////////////////////////////////// - loadStep.setInputRecordPage(transformStep.getOutputRecordPage()); - loadStep.setOutputRecordPage(new ArrayList<>()); - loadStep.run(runBackendStepInput, runBackendStepOutput); + streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, streamedBackendStepOutput.getRecords()); + streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput); + + loadStep.run(streamedBackendStepInput, streamedBackendStepOutput); /////////////////////////////////////////////////////// // copy a small number of records to the output list // /////////////////////////////////////////////////////// int i = 0; - while(loadedRecordList.size() < PROCESS_OUTPUT_RECORD_LIST_LIMIT && i < loadStep.getOutputRecordPage().size()) + while(loadedRecordList.size() < PROCESS_OUTPUT_RECORD_LIST_LIMIT && i < streamedBackendStepOutput.getRecords().size()) { - loadedRecordList.add(loadStep.getOutputRecordPage().get(i++)); + loadedRecordList.add(streamedBackendStepOutput.getRecords().get(i++)); } currentRowCount += qRecords.size(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java index 74367ec4..7dd6c185 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -151,16 +151,21 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe /////////////////////////////////// List qRecords = recordPipe.consumeAvailableRecords(); + /////////////////////////////////////////////////////////////////////// + // make streamed input & output objects from the run input & outputs // + /////////////////////////////////////////////////////////////////////// + StreamedBackendStepInput streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, qRecords); + StreamedBackendStepOutput streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput); + ///////////////////////////////////////////////////// // pass the records through the transform function // ///////////////////////////////////////////////////// - transformStep.setInputRecordPage(qRecords); - transformStep.run(runBackendStepInput, runBackendStepOutput); + transformStep.run(streamedBackendStepInput, streamedBackendStepOutput); //////////////////////////////////////////////////// // add the transformed records to the output list // //////////////////////////////////////////////////// - transformedRecordList.addAll(transformStep.getOutputRecordPage()); + transformedRecordList.addAll(streamedBackendStepOutput.getRecords()); return (qRecords.size()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java index adcd570b..1057c84c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java @@ -128,19 +128,24 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back /////////////////////////////////// List qRecords = recordPipe.consumeAvailableRecords(); + /////////////////////////////////////////////////////////////////////// + // make streamed input & output objects from the run input & outputs // + /////////////////////////////////////////////////////////////////////// + StreamedBackendStepInput streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, qRecords); + StreamedBackendStepOutput streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput); + ///////////////////////////////////////////////////// // pass the records through the transform function // ///////////////////////////////////////////////////// - transformStep.setInputRecordPage(qRecords); - transformStep.run(runBackendStepInput, runBackendStepOutput); + transformStep.run(streamedBackendStepInput, streamedBackendStepOutput); /////////////////////////////////////////////////////// // copy a small number of records to the output list // /////////////////////////////////////////////////////// int i = 0; - while(previewRecordList.size() < PROCESS_OUTPUT_RECORD_LIST_LIMIT && i < transformStep.getOutputRecordPage().size()) + while(previewRecordList.size() < PROCESS_OUTPUT_RECORD_LIST_LIMIT && i < streamedBackendStepOutput.getRecords().size()) { - previewRecordList.add(transformStep.getOutputRecordPage().get(i++)); + previewRecordList.add(streamedBackendStepOutput.getRecords().get(i++)); } currentRowCount += qRecords.size(); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java index 5508520a..62cf8540 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java @@ -347,12 +347,12 @@ public class StreamedETLWithFrontendProcessTest @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - for(QRecord qRecord : getInputRecordPage()) + for(QRecord qRecord : runBackendStepInput.getRecords()) { QRecord newQRecord = new QRecord(); newQRecord.setValue("firstName", "Johnny"); newQRecord.setValue("lastName", qRecord.getValueString("name")); - getOutputRecordPage().add(newQRecord); + runBackendStepOutput.getRecords().add(newQRecord); } } @@ -403,7 +403,7 @@ public class StreamedETLWithFrontendProcessTest @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - for(QRecord qRecord : getInputRecordPage()) + for(QRecord qRecord : runBackendStepInput.getRecords()) { if(qRecord.getValueString("name").equals("Circle")) { @@ -414,7 +414,7 @@ public class StreamedETLWithFrontendProcessTest QRecord newQRecord = new QRecord(); newQRecord.setValue("firstName", "Johnny"); newQRecord.setValue("lastName", qRecord.getValueString("name")); - getOutputRecordPage().add(newQRecord); + runBackendStepOutput.getRecords().add(newQRecord); okSummary.incrementCountAndAddPrimaryKey(qRecord.getValue("id")); } @@ -437,12 +437,12 @@ public class StreamedETLWithFrontendProcessTest @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - for(QRecord qRecord : getInputRecordPage()) + for(QRecord qRecord : runBackendStepInput.getRecords()) { QRecord updatedQRecord = new QRecord(); updatedQRecord.setValue("id", qRecord.getValue("id")); updatedQRecord.setValue("name", "Transformed:" + qRecord.getValueString("name")); - getOutputRecordPage().add(updatedQRecord); + runBackendStepOutput.getRecords().add(updatedQRecord); } } diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java index d0807971..f3833670 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java @@ -76,7 +76,7 @@ public class ClonePeopleTransformStep extends AbstractTransformStep implements P @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - for(QRecord inputPerson : getInputRecordPage()) + for(QRecord inputPerson : runBackendStepInput.getRecords()) { Serializable id = inputPerson.getValue("id"); if("Garret".equals(inputPerson.getValueString("firstName"))) @@ -92,7 +92,7 @@ public class ClonePeopleTransformStep extends AbstractTransformStep implements P QRecord outputPerson = new QRecord(inputPerson); outputPerson.setValue("id", null); outputPerson.setValue("firstName", "Clone of: " + inputPerson.getValueString("firstName")); - getOutputRecordPage().add(outputPerson); + runBackendStepOutput.getRecords().add(outputPerson); if(inputPerson.getValueString("firstName").matches("Clone of.*")) { diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java index 1ea36c1c..a1c5758f 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java @@ -91,7 +91,7 @@ class ClonePeopleTransformStepTest RunBackendStepOutput output = new RunBackendStepOutput(); ClonePeopleTransformStep clonePeopleTransformStep = new ClonePeopleTransformStep(); - clonePeopleTransformStep.setInputRecordPage(queryOutput.getRecords()); + input.setRecords(queryOutput.getRecords()); clonePeopleTransformStep.run(input, output); ArrayList processSummary = clonePeopleTransformStep.getProcessSummary(true);