From 7355a9c8aa7f255544d138df0da3db9e55c2d417 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 24 Aug 2022 10:18:59 -0500 Subject: [PATCH] QQQ-37 Updating test coverage --- .../ExtractViaQueryStep.java | 37 ++-- .../StreamedETLWithFrontendProcessTest.java | 208 ++++++++++++++---- 2 files changed, 186 insertions(+), 59 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java index f843668d..0e1b81ae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java @@ -78,14 +78,7 @@ public class ExtractViaQueryStep extends AbstractExtractStep String queryFilterJson = runBackendStepInput.getValueString("queryFilterJson"); if(queryFilterJson != null) { - try - { - return (JsonUtils.toObject(queryFilterJson, QQueryFilter.class)); - } - catch(IOException e) - { - throw new QException("Error loading query filter from json field", e); - } + return getQueryFilterFromJson(queryFilterJson, "Error loading query filter from json field"); } else if(runBackendStepInput.getCallback() != null && runBackendStepInput.getCallback().getQueryFilter() != null) { @@ -106,20 +99,30 @@ public class ExtractViaQueryStep extends AbstractExtractStep { return (filter); } - if(defaultQueryFilter instanceof String string) + else if(defaultQueryFilter instanceof String string) { - try - { - return (JsonUtils.toObject(string, QQueryFilter.class)); - } - catch(IOException e) - { - throw new QException("Error loading default query filter from json", e); - } + return getQueryFilterFromJson(string, "Error loading default query filter from json"); } } throw (new QException("Could not find query filter for Extract step.")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + private QQueryFilter getQueryFilterFromJson(String queryFilterJson, String message) throws QException + { + try + { + return (JsonUtils.toObject(queryFilterJson, QQueryFilter.class)); + } + catch(IOException e) + { + throw new QException(message, e); + } + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java index ad63741e..2c344a49 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java @@ -2,6 +2,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.io.Serializable; +import java.util.HashMap; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; @@ -11,13 +12,17 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -31,6 +36,17 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class StreamedETLWithFrontendProcessTest { + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + /******************************************************************************* ** @@ -43,7 +59,12 @@ class StreamedETLWithFrontendProcessTest //////////////////////////////////////////////////////// // define the process - an ELT from Shapes to Persons // //////////////////////////////////////////////////////// - QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData(TestUtils.TABLE_NAME_SHAPE, TestUtils.TABLE_NAME_PERSON, ExtractViaQueryStep.class, TestTransformStep.class, LoadViaInsertStep.class); + QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + TestUtils.TABLE_NAME_SHAPE, + TestUtils.TABLE_NAME_PERSON, + ExtractViaQueryStep.class, + TestTransformShapeToPersonStep.class, + LoadViaInsertStep.class); process.setTableName(TestUtils.TABLE_NAME_SHAPE); instance.addProcess(process); @@ -57,15 +78,7 @@ class StreamedETLWithFrontendProcessTest ///////////////////// // run the process // ///////////////////// - RunProcessInput request = new RunProcessInput(instance); - request.setSession(TestUtils.getMockSession()); - request.setProcessName(process.getName()); - request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); - request.setCallback(new Callback()); - - RunProcessOutput result = new RunProcessAction().execute(request); - assertNotNull(result); - assertTrue(result.getException().isEmpty()); + runProcess(instance, process); List postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_PERSON); assertThat(postList) @@ -80,33 +93,31 @@ class StreamedETLWithFrontendProcessTest ** *******************************************************************************/ @Test - void testBig() throws QException + void testSimpleSmallQueryTransformUpdate() throws QException { QInstance instance = TestUtils.defineInstance(); - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // define the process - an ELT from Persons to Persons - using the mock backend, and set to do many many records // - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_PERSON, ExtractViaQueryWithCustomLimitStep.class, TestTransformStep.class, LoadViaInsertStep.class); + //////////////////////////////////////////////////////// + // define the process - an ELT from Shapes to Shapes // + //////////////////////////////////////////////////////// + QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + TestUtils.TABLE_NAME_SHAPE, + TestUtils.TABLE_NAME_SHAPE, + ExtractViaQueryStep.class, + TestTransformUpdateShapeStep.class, + LoadViaUpdateStep.class); process.setTableName(TestUtils.TABLE_NAME_SHAPE); instance.addProcess(process); - ///////////////////// - // run the process // - ///////////////////// - RunProcessInput request = new RunProcessInput(instance); - request.setSession(TestUtils.getMockSession()); - request.setProcessName(process.getName()); - request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); - request.setCallback(new Callback()); + TestUtils.insertDefaultShapes(instance); - RunProcessOutput result = new RunProcessAction().execute(request); - assertNotNull(result); - assertTrue(result.getException().isEmpty()); + runProcess(instance, process); - assertEquals(new ExtractViaQueryWithCustomLimitStep().getLimit(), result.getValues().get(StreamedETLProcess.FIELD_RECORD_COUNT)); - - // todo what can we assert? + List postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_SHAPE); + for(String name : new String[] { "Circle", "Triangle", "Square" }) + { + assertThat(postList).as("Should have transformed and updated " + name).anyMatch(qr -> qr.getValue("name").equals("Transformed:" + name)); + } } @@ -114,7 +125,106 @@ class StreamedETLWithFrontendProcessTest /******************************************************************************* ** *******************************************************************************/ - public static class TestTransformStep extends AbstractTransformStep + @Test + void testWithDefaultQueryFilter() throws QException + { + QInstance instance = TestUtils.defineInstance(); + + //////////////////////////////////////////////////////// + // define the process - an ELT from Shapes to Shapes // + //////////////////////////////////////////////////////// + QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + TestUtils.TABLE_NAME_SHAPE, + TestUtils.TABLE_NAME_SHAPE, + ExtractViaQueryStep.class, + TestTransformUpdateShapeStep.class, + LoadViaUpdateStep.class); + process.setTableName(TestUtils.TABLE_NAME_SHAPE); + instance.addProcess(process); + + TestUtils.insertDefaultShapes(instance); + + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter().withCriteria(new QFilterCriteria("name", QCriteriaOperator.EQUALS, List.of("Square")))); + RunProcessOutput output = runProcess(instance, process, values, null); + assertEquals(1, output.getValues().get(StreamedETLProcess.FIELD_RECORD_COUNT)); + + List postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_SHAPE); + for(String name : new String[] { "Square" }) + { + assertThat(postList).as("Should have transformed and updated " + name).anyMatch(qr -> qr.getValue("name").equals("Transformed:" + name)); + } + + for(String name : new String[] { "Circle", "Triangle"}) + { + assertThat(postList).as("Should not have transformed and updated " + name).anyMatch(qr -> qr.getValue("name").equals(name)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBig() throws QException + { + QInstance instance = TestUtils.defineInstance(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // define the process - an ELT from Persons to Persons - using the mock backend, and set to do very many records // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + TestUtils.TABLE_NAME_PERSON, + TestUtils.TABLE_NAME_PERSON, + ExtractViaQueryWithCustomLimitStep.class, + TestTransformShapeToPersonStep.class, + LoadViaInsertStep.class); + process.setTableName(TestUtils.TABLE_NAME_SHAPE); + instance.addProcess(process); + + ///////////////////// + // run the process // + ///////////////////// + RunProcessOutput output = runProcess(instance, process); + assertEquals(new ExtractViaQueryWithCustomLimitStep().getLimit(), output.getValues().get(StreamedETLProcess.FIELD_RECORD_COUNT)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private RunProcessOutput runProcess(QInstance instance, QProcessMetaData process) throws QException + { + return (runProcess(instance, process, new HashMap<>(), new Callback())); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + private RunProcessOutput runProcess(QInstance instance, QProcessMetaData process, Map values, QProcessCallback callback) throws QException + { + RunProcessInput request = new RunProcessInput(instance); + request.setSession(TestUtils.getMockSession()); + request.setProcessName(process.getName()); + request.setValues(values); + request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + request.setCallback(callback); + + RunProcessOutput output = new RunProcessAction().execute(request); + assertNotNull(output); + assertTrue(output.getException().isEmpty()); + return (output); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestTransformShapeToPersonStep extends AbstractTransformStep { /******************************************************************************* @@ -136,6 +246,31 @@ class StreamedETLWithFrontendProcessTest + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestTransformUpdateShapeStep extends AbstractTransformStep + { + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + for(QRecord qRecord : getInputRecordPage()) + { + QRecord updatedQRecord = new QRecord(); + updatedQRecord.setValue("id", qRecord.getValue("id")); + updatedQRecord.setValue("name", "Transformed:" + qRecord.getValueString("name")); + getOutputRecordPage().add(updatedQRecord); + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -149,17 +284,6 @@ class StreamedETLWithFrontendProcessTest { return (new QQueryFilter()); } - - - - /******************************************************************************* - ** Get the field values for this callback. - *******************************************************************************/ - @Override - public Map getFieldValues(List fields) - { - return (null); - } }