Checkpoint - working versions of streamed with frontend processes, with validation

This commit is contained in:
2022-08-29 13:33:35 -05:00
parent 5f8f063b99
commit cb22f86793
31 changed files with 1253 additions and 134 deletions

View File

@ -23,24 +23,34 @@ package com.kingsrook.qqq.backend.core.actions.processes;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
import com.kingsrook.qqq.backend.core.state.StateType;
import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
@ -56,6 +66,9 @@ import static org.junit.jupiter.api.Assertions.fail;
*******************************************************************************/
public class RunProcessTest
{
private static final Logger LOG = LogManager.getLogger(RunProcessTest.class);
/*******************************************************************************
**
@ -85,7 +98,7 @@ public class RunProcessTest
@Test
public void testBreakOnFrontendSteps() throws QException
{
TestCallback callback = new TestCallback();
TestCallback callback = new TestCallback();
QInstance instance = TestUtils.defineInstance();
RunProcessInput request = new RunProcessInput(instance);
String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE;
@ -130,7 +143,7 @@ public class RunProcessTest
@Test
public void testSkipFrontendSteps() throws QException
{
TestCallback callback = new TestCallback();
TestCallback callback = new TestCallback();
QInstance instance = TestUtils.defineInstance();
RunProcessInput request = new RunProcessInput(instance);
String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE;
@ -154,7 +167,7 @@ public class RunProcessTest
@Test
public void testFailOnFrontendSteps()
{
TestCallback callback = new TestCallback();
TestCallback callback = new TestCallback();
QInstance instance = TestUtils.defineInstance();
RunProcessInput request = new RunProcessInput(instance);
String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE;
@ -188,7 +201,8 @@ public class RunProcessTest
////////////////////////////////////////////////////////////////////////////////
RunProcessInput runProcessInput = new RunProcessInput();
UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS);
ProcessState processState = new RunProcessAction().primeProcessState(runProcessInput, stateKey);
QProcessMetaData process = TestUtils.defineInstance().getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE);
ProcessState processState = new RunProcessAction().primeProcessState(runProcessInput, stateKey, process);
assertNotNull(processState);
}
@ -206,10 +220,11 @@ public class RunProcessTest
RunProcessInput runProcessInput = new RunProcessInput();
runProcessInput.setStartAfterStep("setupStep");
UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS);
QProcessMetaData process = TestUtils.defineInstance().getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE);
assertThrows(QException.class, () ->
{
new RunProcessAction().primeProcessState(runProcessInput, stateKey);
new RunProcessAction().primeProcessState(runProcessInput, stateKey, process);
});
}
@ -238,7 +253,8 @@ public class RunProcessTest
oldProcessState.getValues().put("foo", "fubu");
RunProcessAction.getStateProvider().put(stateKey, oldProcessState);
ProcessState primedProcessState = new RunProcessAction().primeProcessState(runProcessInput, stateKey);
QProcessMetaData process = TestUtils.defineInstance().getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE);
ProcessState primedProcessState = new RunProcessAction().primeProcessState(runProcessInput, stateKey, process);
assertEquals("myValue", primedProcessState.getValues().get("key"));
/////////////////////////////////////////////////////////////////////////////////////////////
@ -250,6 +266,163 @@ public class RunProcessTest
/*******************************************************************************
** Test a simple version of custom routing, where we just add a frontend step.
*******************************************************************************/
@Test
void testCustomRoutingAddFrontendStep() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
QStepMetaData back1 = new QBackendStepMetaData()
.withName("back1")
.withCode(new QCodeReference(BackendStepThatMayAddFrontendStep.class));
QStepMetaData front1 = new QFrontendStepMetaData()
.withName("front1");
String processName = "customRouting";
qInstance.addProcess(new QProcessMetaData()
.withName(processName)
.withStepList(List.of(
back1
//////////////////////////////////////
// only put back1 in the step list. //
//////////////////////////////////////
))
.addOptionalStep(front1)
);
////////////////////////////////////////////////////////////
// make sure that if we run by default, we get to the end //
////////////////////////////////////////////////////////////
RunProcessInput request = new RunProcessInput(qInstance);
request.setSession(TestUtils.getMockSession());
request.setProcessName(processName);
request.setCallback(new TestCallback());
request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
RunProcessOutput result = new RunProcessAction().execute(request);
assertThat(result.getProcessState().getNextStepName()).isEmpty();
/////////////////////////////////////////////////////////////////////////////////////////////
// now run again, with the field set to cause the front1 step to be added to the step list //
/////////////////////////////////////////////////////////////////////////////////////////////
request.addValue("shouldAddFrontendStep", true);
result = new RunProcessAction().execute(request);
assertThat(result.getProcessState().getNextStepName()).hasValue("front1");
}
/*******************************************************************************
**
*******************************************************************************/
public static class BackendStepThatMayAddFrontendStep implements BackendStep
{
/*******************************************************************************
** Execute the backend step - using the request as input, and the result as output.
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
LOG.info("Running " + getClass().getSimpleName());
if(runBackendStepInput.getValue("shouldAddFrontendStep") != null)
{
List<String> stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList());
stepList.add("front1");
runBackendStepOutput.getProcessState().setStepList(stepList);
}
}
}
/*******************************************************************************
** Test a version of custom routing, where we remove steps
*******************************************************************************/
@Test
void testCustomRoutingRemoveSteps() throws QException
{
QInstance qInstance = TestUtils.defineInstance();
QStepMetaData back1 = new QBackendStepMetaData()
.withName("back1")
.withCode(new QCodeReference(BackendStepThatMayRemoveFrontendStep.class));
QStepMetaData front1 = new QFrontendStepMetaData()
.withName("front1");
QStepMetaData back2 = new QBackendStepMetaData()
.withName("back2")
.withCode(new QCodeReference(NoopBackendStep.class));
QStepMetaData front2 = new QFrontendStepMetaData()
.withName("front2");
String processName = "customRouting";
qInstance.addProcess(new QProcessMetaData()
.withName(processName)
.withStepList(List.of(
back1,
front1,
back2,
front2
))
);
/////////////////////////////////////////////////////////////////////////////
// make sure that if we run by default, we get stop on both frontend steps //
/////////////////////////////////////////////////////////////////////////////
RunProcessInput request = new RunProcessInput(qInstance);
request.setSession(TestUtils.getMockSession());
request.setProcessName(processName);
request.setCallback(new TestCallback());
request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
RunProcessOutput result = new RunProcessAction().execute(request);
assertThat(result.getProcessState().getNextStepName()).hasValue("front1");
request.setStartAfterStep("front1");
result = new RunProcessAction().execute(request);
assertThat(result.getProcessState().getNextStepName()).hasValue("front2");
/////////////////////////////////////////////////////////////////////////////////////////////////
// now run again, with the field set to cause the front1 step to be removed from the step list //
/////////////////////////////////////////////////////////////////////////////////////////////////
request.setStartAfterStep(null);
request.addValue("shouldRemoveFrontendStep", true);
result = new RunProcessAction().execute(request);
assertThat(result.getProcessState().getNextStepName()).hasValue("front2");
}
/*******************************************************************************
**
*******************************************************************************/
public static class BackendStepThatMayRemoveFrontendStep implements BackendStep
{
/*******************************************************************************
** Execute the backend step - using the request as input, and the result as output.
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
LOG.info("Running " + getClass().getSimpleName());
if(runBackendStepInput.getValue("shouldRemoveFrontendStep") != null)
{
List<String> stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList());
stepList.removeIf(s -> s.equals("front1"));
runBackendStepOutput.getProcessState().setStepList(stepList);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -287,4 +460,24 @@ public class RunProcessTest
return (rs);
}
}
/*******************************************************************************
**
*******************************************************************************/
public static class NoopBackendStep implements BackendStep
{
/*******************************************************************************
** Execute the backend step - using the request as input, and the result as output.
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
LOG.info("Running " + getClass().getSimpleName());
}
}
}

View File

@ -64,7 +64,8 @@ class ReportActionTest
runReport(recordCount, filename, ReportFormat.CSV, false);
File file = new File(filename);
File file = new File(filename);
@SuppressWarnings("unchecked")
List<String> fileLines = FileUtils.readLines(file, StandardCharsets.UTF_8.name());
assertEquals(recordCount + 1, fileLines.size());
assertTrue(file.delete());
@ -85,7 +86,8 @@ class ReportActionTest
runReport(recordCount, filename, ReportFormat.CSV, false);
File file = new File(filename);
File file = new File(filename);
@SuppressWarnings("unchecked")
List<String> fileLines = FileUtils.readLines(file, StandardCharsets.UTF_8.name());
assertEquals(recordCount + 1, fileLines.size());
assertTrue(file.delete());

View File

@ -2,16 +2,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@ -155,7 +158,7 @@ class StreamedETLWithFrontendProcessTest
assertThat(postList).as("Should have transformed and updated " + name).anyMatch(qr -> qr.getValue("name").equals("Transformed:" + name));
}
for(String name : new String[] { "Circle", "Triangle"})
for(String name : new String[] { "Circle", "Triangle" })
{
assertThat(postList).as("Should not have transformed and updated " + name).anyMatch(qr -> qr.getValue("name").equals(name));
}
@ -192,6 +195,47 @@ class StreamedETLWithFrontendProcessTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithValidationStep() throws QException
{
QInstance instance = TestUtils.defineInstance();
////////////////////////////////////////////////////////
// define the process - an ELT from Shapes to Persons //
////////////////////////////////////////////////////////
QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData(
TestUtils.TABLE_NAME_SHAPE,
TestUtils.TABLE_NAME_PERSON,
ExtractViaQueryStep.class,
TestTransformShapeToPersonWithValidationStep.class,
LoadViaInsertStep.class);
process.setTableName(TestUtils.TABLE_NAME_SHAPE);
instance.addProcess(process);
///////////////////////////////////////////////////////
// switch the person table to use the memory backend //
///////////////////////////////////////////////////////
instance.getTable(TestUtils.TABLE_NAME_PERSON).setBackendName(TestUtils.MEMORY_BACKEND_NAME);
TestUtils.insertDefaultShapes(instance);
/////////////////////
// run the process // todo - don't skip FE steps
/////////////////////
runProcess(instance, process);
List<QRecord> postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_PERSON);
assertThat(postList)
.as("Should have inserted Circle").anyMatch(qr -> qr.getValue("lastName").equals("Circle"))
.as("Should have inserted Triangle").anyMatch(qr -> qr.getValue("lastName").equals("Triangle"))
.as("Should have inserted Square").anyMatch(qr -> qr.getValue("lastName").equals("Square"));
}
/*******************************************************************************
**
*******************************************************************************/
@ -201,6 +245,7 @@ class StreamedETLWithFrontendProcessTest
}
/*******************************************************************************
**
*******************************************************************************/
@ -246,6 +291,63 @@ class StreamedETLWithFrontendProcessTest
/*******************************************************************************
**
*******************************************************************************/
public static class TestTransformShapeToPersonWithValidationStep extends AbstractTransformStep implements ProcessSummaryProviderInterface
{
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK, 0, "can be transformed into a Person");
private ProcessSummaryLine notAPolygonSummary = new ProcessSummaryLine(Status.OK, 0, "cannot be transformed, because they are not a Polygon");
/*******************************************************************************
**
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLine> getProcessSummary(boolean isForResultScreen)
{
if(isForResultScreen)
{
okSummary.setMessage("were transformed into a Person");
}
ArrayList<ProcessSummaryLine> summaryList = new ArrayList<>();
summaryList.add(okSummary);
summaryList.add(notAPolygonSummary);
return (summaryList);
}
/*******************************************************************************
** Execute the backend step - using the request as input, and the result as output.
**
*******************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
for(QRecord qRecord : getInputRecordPage())
{
if(qRecord.getValueString("name").equals("Circle"))
{
notAPolygonSummary.incrementCountAndAddPrimaryKey(qRecord.getValue("id"));
}
else
{
QRecord newQRecord = new QRecord();
newQRecord.setValue("firstName", "Johnny");
newQRecord.setValue("lastName", qRecord.getValueString("name"));
getOutputRecordPage().add(newQRecord);
okSummary.incrementCountAndAddPrimaryKey(qRecord.getValue("id"));
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/

View File

@ -328,7 +328,7 @@ public class TestUtils
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
.withField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
)
.withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING))))
);
@ -366,7 +366,7 @@ public class TestUtils
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
.withField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
)
.withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING))))
)
@ -403,7 +403,7 @@ public class TestUtils
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("age", QFieldType.INTEGER)))
.withField(new QFieldMetaData("age", QFieldType.INTEGER)))
.withFieldList(List.of(
new QFieldMetaData("minAge", QFieldType.INTEGER),
new QFieldMetaData("maxAge", QFieldType.INTEGER)))))
@ -418,7 +418,7 @@ public class TestUtils
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName(TABLE_NAME_PERSON)
.addField(new QFieldMetaData("newAge", QFieldType.INTEGER)))));
.withField(new QFieldMetaData("newAge", QFieldType.INTEGER)))));
}