QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 05:30:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

sprint-14: initial checkin of basepull capability on processes

Browse Source
This commit is contained in:
Tim Chamberlain
2022-10-24 17:06:11 -05:00
parent 44537e182d
commit 128c379f10
11 changed files with 658 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java
View File

@ -23,19 +23,27 @@ package com.kingsrook.qqq.backend.core.actions.processes;
import java.io.Serializable;
import java.time.DayOfWeek;
import java.time.Instant;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@ -47,6 +55,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackend
import com.kingsrook.qqq.backend.core.state.StateType;
import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.Test;
@ -70,6 +79,80 @@ public class RunProcessTest
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testBasepull() throws QException
{
TestCallback callback = new TestCallback();
RunProcessInput request = new RunProcessInput(TestUtils.defineInstance());
request.setSession(TestUtils.getMockSession());
request.setProcessName(TestUtils.PROCESS_NAME_BASEPULL);
request.setCallback(callback);
RunProcessOutput result = new RunProcessAction().execute(request);
assertNotNull(result);
//////////////////////////////////////////////////////////////////////////////////////////
// get the last run time and 'this' run time - because the definition states that if no //
// rows found, the last runtime timestamp should be for 24 hours ago //
//////////////////////////////////////////////////////////////////////////////////////////
Instant lastRunTime = (Instant) result.getValues().get(RunProcessAction.BASEPULL_LAST_RUNTIME_KEY);
Instant thisRunTime = (Instant) result.getValues().get(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY);
assertTrue(thisRunTime.isAfter(lastRunTime), "new run time should be after last run time.");
DayOfWeek lastRunTimeDayOfWeek = lastRunTime.atZone(ZoneId.systemDefault()).getDayOfWeek();
DayOfWeek thisRunTimeDayOfWeek = thisRunTime.atZone(ZoneId.systemDefault()).getDayOfWeek();
thisRunTimeDayOfWeek = thisRunTimeDayOfWeek.minus(1);
assertEquals(lastRunTimeDayOfWeek.getValue(), thisRunTimeDayOfWeek.getValue(), "last and this run times should be the same day after subtracting a day");
///////////////////////////////////////////////
// make sure new stamp stored in backend too //
///////////////////////////////////////////////
assertEquals(thisRunTime, getBasepullLastRunTime(), "last run time should be properly stored in backend");
////////////////////////////////////////////////////
// run the process one more time and check values //
////////////////////////////////////////////////////
result = new RunProcessAction().execute(request);
assertNotNull(result);
////////////////////////////////
// this should still be after //
////////////////////////////////
lastRunTime = (Instant) result.getValues().get(RunProcessAction.BASEPULL_LAST_RUNTIME_KEY);
thisRunTime = (Instant) result.getValues().get(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY);
assertTrue(thisRunTime.isAfter(lastRunTime), "new run time should be after last run time.");
///////////////////////////////////////////////
// make sure new stamp stored in backend too //
///////////////////////////////////////////////
assertEquals(thisRunTime, getBasepullLastRunTime(), "last run time should be properly stored in backend");
}
/*******************************************************************************
**
*******************************************************************************/
private Instant getBasepullLastRunTime() throws QException
{
QueryInput queryInput = new QueryInput(TestUtils.defineInstance());
queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria()
.withFieldName(TestUtils.BASEPULL_KEY_FIELD_NAME)
.withOperator(QCriteriaOperator.EQUALS)
.withValues(List.of(TestUtils.PROCESS_NAME_BASEPULL))));
queryInput.setSession(TestUtils.getMockSession());
queryInput.setTableName(TestUtils.TABLE_NAME_BASEPULL);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertNotNull(queryOutput);
assertEquals(1, queryOutput.getRecords().size(), "Should have one record");
return (ValueUtils.getValueAsInstant(queryOutput.getRecords().get(0).getValue(TestUtils.BASEPULL_LAST_RUN_TIME_FIELD_NAME)));
}
/*******************************************************************************
**
*******************************************************************************/

65
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -89,6 +89,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.Mem
import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -116,9 +117,11 @@ public class TestUtils
public static final String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive";
public static final String PROCESS_NAME_INCREASE_BIRTHDATE = "increaseBirthdate";
public static final String PROCESS_NAME_ADD_TO_PEOPLES_AGE = "addToPeoplesAge";
public static final String PROCESS_NAME_BASEPULL = "basepullTest";
public static final String TABLE_NAME_PERSON_FILE = "personFile";
public static final String TABLE_NAME_PERSON_MEMORY = "personMemory";
public static final String TABLE_NAME_ID_AND_NAME_ONLY = "idAndNameOnly";
public static final String TABLE_NAME_BASEPULL = "basepullTest";
public static final String POSSIBLE_VALUE_SOURCE_STATE = "state"; // enum-type
public static final String POSSIBLE_VALUE_SOURCE_SHAPE = "shape"; // table-type
@ -128,6 +131,9 @@ public class TestUtils
public static final String POLLING_AUTOMATION = "polling";
public static final String DEFAULT_QUEUE_PROVIDER = "defaultQueueProvider";
public static final String BASEPULL_KEY_FIELD_NAME = "processName";
public static final String BASEPULL_LAST_RUN_TIME_FIELD_NAME = "lastRunTime";
/*******************************************************************************
@ -146,6 +152,7 @@ public class TestUtils
qInstance.addTable(definePersonMemoryTable());
qInstance.addTable(defineTableIdAndNameOnly());
qInstance.addTable(defineTableShape());
qInstance.addTable(defineTableBasepull());
qInstance.addPossibleValueSource(defineAutomationStatusPossibleValueSource());
qInstance.addPossibleValueSource(defineStatesPossibleValueSource());
@ -158,6 +165,7 @@ public class TestUtils
qInstance.addProcess(new BasicETLProcess().defineProcessMetaData());
qInstance.addProcess(new StreamedETLProcess().defineProcessMetaData());
qInstance.addProcess(defineProcessIncreasePersonBirthdate());
qInstance.addProcess(defineProcessBasepull());
qInstance.addAutomationProvider(definePollingAutomationProvider());
@ -486,6 +494,26 @@ public class TestUtils
/*******************************************************************************
** Define a basepullTable
*******************************************************************************/
public static QTableMetaData defineTableBasepull()
{
return (new QTableMetaData()
.withName(TABLE_NAME_BASEPULL)
.withLabel("Basepull Test")
.withPrimaryKeyField("id")
.withBackendName(MEMORY_BACKEND_NAME)
.withFields(TestUtils.defineTablePerson().getFields()))
.withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date").withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date").withIsEditable(false))
.withField(new QFieldMetaData(BASEPULL_KEY_FIELD_NAME, QFieldType.STRING).withBackendName("process_name").withIsRequired(true))
.withField(new QFieldMetaData(BASEPULL_LAST_RUN_TIME_FIELD_NAME, QFieldType.DATE_TIME).withBackendName("last_run_time").withIsRequired(true));
}
/*******************************************************************************
** Define a 3nd version of the 'person' table, backed by the in-memory backend
*******************************************************************************/
@ -732,6 +760,43 @@ public class TestUtils
/*******************************************************************************
** Define a sample basepull process
*******************************************************************************/
private static QProcessMetaData defineProcessBasepull()
{
return new QProcessMetaData()
.withBasepullConfiguration(new BasepullConfiguration()
.withKeyField(BASEPULL_KEY_FIELD_NAME)
.withLastRunTimeFieldName(BASEPULL_LAST_RUN_TIME_FIELD_NAME)
.withHoursBackForInitialTimestamp(24)
.withKeyValue(PROCESS_NAME_BASEPULL)
.withTableName(defineTableBasepull().getName()))
.withName(PROCESS_NAME_BASEPULL)
.withTableName(TABLE_NAME_PERSON)
.addStep(new QBackendStepMetaData()
.withName("prepare")
.withCode(new QCodeReference()
.withName(MockBackendStep.class.getName())
.withCodeType(QCodeType.JAVA)
.withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context?
.withInputData(new QFunctionInputMetaData()
.withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON))
.withFieldList(List.of(
new QFieldMetaData("greetingPrefix", QFieldType.STRING),
new QFieldMetaData("greetingSuffix", QFieldType.STRING)
)))
.withOutputMetaData(new QFunctionOutputMetaData()
.withRecordListMetaData(new QRecordListMetaData()
.withTableName(TABLE_NAME_PERSON)
.withField(new QFieldMetaData("fullGreeting", QFieldType.STRING))
)
.withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING))))
);
}
/*******************************************************************************
**
*******************************************************************************/