QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 21:20:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-936 - scheduling updates:

Browse Source 
- move queues & automations to be scheduled (only) at the lower-level (per-queue, per-table) - not at the higher "provider" levels.
- update quartz to delete jobs which are no-longer active, at end of QScheduleManager's setup
-
This commit is contained in:
darin.kelkhoff
2024-03-13 12:08:13 -05:00
parent 246984892a
commit 3265d6d842
18 changed files with 385 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java
View File

@ -575,8 +575,8 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
@Test
void testLoadingRecordTypesToEnsureClassCoverage()
{
new PollingAutomationPerTableRunner.TableActions(null, null).noopToFakeTestCoverage();
new PollingAutomationPerTableRunner.ShardedTableActions(null, null, null, null, null).noopToFakeTestCoverage();
new PollingAutomationPerTableRunner.TableActions(null, null, null).noopToFakeTestCoverage();
new PollingAutomationPerTableRunner.ShardedTableActions(null, null, null, null, null, null).noopToFakeTestCoverage();
}

16
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java
View File

@ -469,19 +469,19 @@ public class QInstanceValidatorTest extends BaseTest
assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("UTC")));
assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("America/New_York")));
//////////////////////////////////////////////////////////////////
// make sure automation providers get their schedules validated //
//////////////////////////////////////////////////////////////////
assertValidationFailureReasons((qInstance) -> qInstance.getAutomationProvider(TestUtils.POLLING_AUTOMATION).withSchedule(baseScheduleMetaData.get()
///////////////////////////////////////////////////////////////
// make sure table automations get their schedules validated //
///////////////////////////////////////////////////////////////
assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().withSchedule(baseScheduleMetaData.get()
.withSchedulerName(null)
.withCronExpression(validCronString)
.withCronTimeZoneId("UTC")),
"is missing a scheduler name");
/////////////////////////////////////////////////////////////
// make sure queue providers get their schedules validated //
/////////////////////////////////////////////////////////////
assertValidationFailureReasons((qInstance) -> ((SQSQueueProviderMetaData)qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER)).withSchedule(baseScheduleMetaData.get()
////////////////////////////////////////////////////
// make sure queues get their schedules validated //
////////////////////////////////////////////////////
assertValidationFailureReasons((qInstance) -> (qInstance.getQueue(TestUtils.TEST_SQS_QUEUE)).withSchedule(baseScheduleMetaData.get()
.withSchedulerName(null)
.withCronExpression(validCronString)
.withCronTimeZoneId("UTC")),

86
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java
View File

@ -43,7 +43,9 @@ import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import org.apache.logging.log4j.Level;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.quartz.SchedulerException;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -59,7 +61,27 @@ class QuartzSchedulerTest extends BaseTest
@AfterEach
void afterEach()
{
QScheduleManager.getInstance().unInit();
try
{
QScheduleManager.getInstance().unInit();
}
catch(IllegalStateException ise)
{
/////////////////////////////////////////////////////////////////
// ok, might just mean that this test didn't init the instance //
/////////////////////////////////////////////////////////////////
}
try
{
QuartzScheduler.getInstance().unInit();
}
catch(IllegalStateException ise)
{
/////////////////////////////////////////////////////////////////
// ok, might just mean that this test didn't init the instance //
/////////////////////////////////////////////////////////////////
}
}
@ -84,15 +106,7 @@ class QuartzSchedulerTest extends BaseTest
//////////////////////////////////////////
// add a process we can run and observe //
//////////////////////////////////////////
qInstance.addProcess(new QProcessMetaData()
.withName("testScheduledProcess")
.withSchedule(new QScheduleMetaData()
.withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)
.withRepeatMillis(2)
.withInitialDelaySeconds(0))
.withStepList(List.of(new QBackendStepMetaData()
.withName("step")
.withCode(new QCodeReference(BasicStep.class)))));
qInstance.addProcess(buildTestProcess("testScheduledProcess"));
//////////////////////////////////////////////////////////////////////////////
// start the schedule manager, which will schedule things, and start quartz //
@ -108,7 +122,7 @@ class QuartzSchedulerTest extends BaseTest
qScheduleManager.stopAsync();
System.out.println("Ran: " + BasicStep.counter + " times");
assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice (but only ran [" + BasicStep.counter + "] time(s).");
assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice (but only ran [" + BasicStep.counter + "] time(s)).");
//////////////////////////////////////////////////////
// make sure poller ran, and didn't issue any warns //
@ -132,6 +146,56 @@ class QuartzSchedulerTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
private static QProcessMetaData buildTestProcess(String name)
{
return new QProcessMetaData()
.withName(name)
.withSchedule(new QScheduleMetaData()
.withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)
.withRepeatMillis(2)
.withInitialDelaySeconds(0))
.withStepList(List.of(new QBackendStepMetaData()
.withName("step")
.withCode(new QCodeReference(BasicStep.class))));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testRemovingNoLongerNeededJobsDuringSetupSchedules() throws SchedulerException
{
QInstance qInstance = QContext.getQInstance();
QuartzTestUtils.setupInstanceForQuartzTests();
////////////////////////////
// put two jobs in quartz //
////////////////////////////
QProcessMetaData test1 = buildTestProcess("test1");
QProcessMetaData test2 = buildTestProcess("test2");
qInstance.addProcess(test1);
qInstance.addProcess(test2);
QuartzScheduler quartzScheduler = QuartzScheduler.initInstance(qInstance, QuartzTestUtils.QUARTZ_SCHEDULER_NAME, QuartzTestUtils.getQuartzProperties(), () -> QContext.getQSession());
quartzScheduler.setupProcess(test1, null, test1.getSchedule(), false);
quartzScheduler.setupProcess(test2, null, test2.getSchedule(), false);
quartzScheduler.startOfSetupSchedules();
quartzScheduler.setupProcess(test1, null, test1.getSchedule(), false);
quartzScheduler.endOfSetupSchedules();
List<QuartzJobAndTriggerWrapper> quartzJobAndTriggerWrappers = quartzScheduler.queryQuartz();
assertEquals(1, quartzJobAndTriggerWrappers.size());
assertEquals("test1", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName());
}
/*******************************************************************************
**
*******************************************************************************/

17
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java
View File

@ -26,7 +26,6 @@ import java.util.List;
import java.util.Properties;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData;
import org.quartz.SchedulerException;
@ -43,7 +42,7 @@ public class QuartzTestUtils
/*******************************************************************************
**
*******************************************************************************/
private static Properties getQuartzProperties()
public static Properties getQuartzProperties()
{
Properties quartzProperties = new Properties();
quartzProperties.put("org.quartz.scheduler.instanceName", QUARTZ_SCHEDULER_NAME);
@ -76,12 +75,16 @@ public class QuartzTestUtils
////////////////////////////////////////////////////////////////////////////////
// set the queue providers & automation providers to use the quartz scheduler //
////////////////////////////////////////////////////////////////////////////////
qInstance.getAutomationProviders().values()
.forEach(ap -> ap.getSchedule().setSchedulerName(QUARTZ_SCHEDULER_NAME));
qInstance.getQueueProviders().values()
.forEach(qp -> ((SQSQueueProviderMetaData) qp).getSchedule().setSchedulerName(QUARTZ_SCHEDULER_NAME));
qInstance.getTables().values().forEach(t ->
{
if(t.getAutomationDetails() != null)
{
t.getAutomationDetails().getSchedule().setSchedulerName(QUARTZ_SCHEDULER_NAME);
}
});
qInstance.getQueues().values()
.forEach(q -> q.getSchedule().setSchedulerName(QUARTZ_SCHEDULER_NAME));
}

24
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java
View File

@ -89,8 +89,28 @@ class QuartzJobsProcessTest extends BaseTest
@AfterEach
void afterEach()
{
QScheduleManager.getInstance().stop();
QScheduleManager.getInstance().unInit();
try
{
QScheduleManager.getInstance().stop();
QScheduleManager.getInstance().unInit();
}
catch(IllegalStateException ise)
{
/////////////////////////////////////////////////////////////////
// ok, might just mean that this test didn't init the instance //
/////////////////////////////////////////////////////////////////
}
try
{
QuartzScheduler.getInstance().unInit();
}
catch(IllegalStateException ise)
{
/////////////////////////////////////////////////////////////////
// ok, might just mean that this test didn't init the instance //
/////////////////////////////////////////////////////////////////
}
}

21
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -180,6 +180,7 @@ public class TestUtils
public static final String SECURITY_KEY_TYPE_INTERNAL_OR_EXTERNAL = "internalOrExternal";
public static final String SIMPLE_SCHEDULER_NAME = "simpleScheduler";
public static final String TEST_SQS_QUEUE = "testSQSQueue";
@ -366,10 +367,7 @@ public class TestUtils
private static QAutomationProviderMetaData definePollingAutomationProvider()
{
return (new PollingAutomationProviderMetaData()
.withName(POLLING_AUTOMATION)
.withSchedule(new QScheduleMetaData()
.withSchedulerName(SIMPLE_SCHEDULER_NAME)
.withRepeatSeconds(60)));
.withName(POLLING_AUTOMATION));
}
@ -746,6 +744,9 @@ public class TestUtils
{
return (new QTableAutomationDetails()
.withProviderName(POLLING_AUTOMATION)
.withSchedule(new QScheduleMetaData()
.withSchedulerName(SIMPLE_SCHEDULER_NAME)
.withRepeatSeconds(60))
.withStatusTracking(new AutomationStatusTracking()
.withType(AutomationStatusTrackingType.FIELD_IN_TABLE)
.withFieldName("qqqAutomationStatus")));
@ -1333,10 +1334,7 @@ public class TestUtils
.withAccessKey(accessKey)
.withSecretKey(secretKey)
.withRegion(region)
.withBaseURL(baseURL)
.withSchedule(new QScheduleMetaData()
.withRepeatSeconds(60)
.withSchedulerName(SIMPLE_SCHEDULER_NAME)));
.withBaseURL(baseURL));
}
@ -1347,10 +1345,13 @@ public class TestUtils
private static QQueueMetaData defineTestSqsQueue()
{
return (new QQueueMetaData()
.withName("testSQSQueue")
.withName(TEST_SQS_QUEUE)
.withProviderName(DEFAULT_QUEUE_PROVIDER)
.withQueueName("test-queue")
.withProcessName(PROCESS_NAME_INCREASE_BIRTHDATE));
.withProcessName(PROCESS_NAME_INCREASE_BIRTHDATE)
.withSchedule(new QScheduleMetaData()
.withRepeatSeconds(60)
.withSchedulerName(SIMPLE_SCHEDULER_NAME)));
}