From 5dc4b26f9ee1129c7ea3a3d17d217ec83bbda3ec Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 Apr 2024 20:18:23 -0500 Subject: [PATCH] Add ScheduleAllNewJobsProcess and supporting methods --- .../core/scheduler/QScheduleManager.java | 35 ++++ .../core/scheduler/QSchedulerInterface.java | 6 + .../processes/ScheduleAllNewJobsProcess.java | 90 +++++++++ .../scheduler/quartz/QuartzScheduler.java | 21 ++ .../scheduler/simple/SimpleScheduler.java | 12 ++ .../ScheduleAllNewJobsProcessTest.java | 179 ++++++++++++++++++ 6 files changed, 343 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcess.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcessTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index a51999ab..3dfec3ae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -191,6 +191,41 @@ public class QScheduleManager + /******************************************************************************* + ** + *******************************************************************************/ + public void setupAllNewSchedules() throws QException + { + if(QContext.getQInstance().getTables().containsKey(ScheduledJob.TABLE_NAME)) + { + List scheduledJobList = new QueryAction() + .execute(new QueryInput(ScheduledJob.TABLE_NAME) + .withIncludeAssociations(true)) + .getRecordEntities(ScheduledJob.class); + + for(ScheduledJob scheduledJob : scheduledJobList) + { + try + { + QSchedulerInterface scheduler = getScheduler(scheduledJob.getSchedulerName()); + BasicSchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(scheduledJob); + SchedulableType schedulableType = qInstance.getSchedulableType(scheduledJob.getType()); + + if(!scheduler.isScheduled(schedulableIdentity, schedulableType)) + { + setupScheduledJob(scheduledJob); + } + } + catch(Exception e) + { + LOG.warn("Error evaluating scheduled job", logPair("id", scheduledJob.getId())); + } + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java index b9f16902..3d99ee0b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java @@ -27,6 +27,7 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; @@ -65,6 +66,11 @@ public interface QSchedulerInterface *******************************************************************************/ void unscheduleSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType); + /******************************************************************************* + ** + *******************************************************************************/ + boolean isScheduled(BasicSchedulableIdentity schedulableIdentity, SchedulableType schedulableType); + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcess.java new file mode 100644 index 00000000..408ec1b8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcess.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; + + +/******************************************************************************* + ** Management process to schedule all new scheduled jobs (in all schedulers). + *******************************************************************************/ +public class ScheduleAllNewJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Schedule all New Scheduled Jobs") + .withIcon(new QIcon("more_time")) + .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to schedule all new jobs."))), + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All new jobs have been scheduled."))))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QScheduleManager.getInstance().setupAllNewSchedules(); + } + catch(Exception e) + { + throw (new QException("Error scheduling new jobs.", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java index f41362c5..b3ab8be8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java @@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaDa import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.memoization.AnyKey; @@ -486,6 +487,26 @@ public class QuartzScheduler implements QSchedulerInterface + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean isScheduled(BasicSchedulableIdentity schedulableIdentity, SchedulableType schedulableType) + { + try + { + JobKey jobKey = new JobKey(schedulableIdentity.getIdentity(), schedulableType.getName()); + return (isJobAlreadyScheduled(jobKey)); + } + catch(Exception e) + { + LOG.warn("Error checking if job is scheduled", logPair("identity", schedulableIdentity)); + return (false); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java index 04ff9a6f..8574fecc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java @@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaDa import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -203,6 +204,17 @@ public class SimpleScheduler implements QSchedulerInterface + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean isScheduled(BasicSchedulableIdentity schedulableIdentity, SchedulableType schedulableType) + { + return (executors.containsKey(schedulableIdentity)); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcessTest.java new file mode 100644 index 00000000..ae4bd15c --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/processes/ScheduleAllNewJobsProcessTest.java @@ -0,0 +1,179 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobsMetaDataProvider; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableSQSQueueRunner; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.quartz.SchedulerException; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for ScheduleAllNewJobsProcess + *******************************************************************************/ +class ScheduleAllNewJobsProcessTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QLogger.deactivateCollectingLoggerForClass(QuartzScheduler.class); + + try + { + QScheduleManager.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + + try + { + QuartzScheduler.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException, SchedulerException + { + try + { + QCollectingLogger quartzSchedulerLog = QLogger.activateCollectingLoggerForClass(QuartzScheduler.class); + + QInstance qInstance = QContext.getQInstance(); + new ScheduledJobsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, ScheduleAllNewJobsProcess.class.getPackageName()); + QuartzTestUtils.setupInstanceForQuartzTests(); + + /////////////////////////////////////////////////////////////////////////////////// + // clear out the customizers that would normally schedule jobs as we insert them // + /////////////////////////////////////////////////////////////////////////////////// + qInstance.getTable(ScheduledJob.TABLE_NAME).withCustomizers(Collections.emptyMap()); + + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> QContext.getQSession()); + qScheduleManager.start(); + + QuartzScheduler quartzScheduler = QuartzScheduler.getInstance(); + List wrappers = quartzScheduler.queryQuartz(); + + ////////////////////////////////////////////// + // make sure nothing is scheduled initially // + ////////////////////////////////////////////// + assertTrue(wrappers.isEmpty()); + + //////////////////////////////////////////////////////////////////////////// + // insert a scheduled job - run schedule-new, make sure it gets scheduled // + //////////////////////////////////////////////////////////////////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(SchedulerTestUtils + .newScheduledJob(ScheduledJobType.PROCESS, Map.of("processName", TestUtils.PROCESS_NAME_GREET_PEOPLE)) + .withLabel("Test job 1") + .withId(null) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME))); + + RunProcessInput input = new RunProcessInput(); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.setProcessName(ScheduleAllNewJobsProcess.class.getSimpleName()); + new RunProcessAction().execute(input); + + /////////////////////////////////////////////////////////////// + // make sure our scheduledJob here got scheduled with quartz // + /////////////////////////////////////////////////////////////// + wrappers = quartzScheduler.queryQuartz(); + assertEquals(1, wrappers.size()); + assertTrue(wrappers.stream().anyMatch(w -> w.jobDetail().getKey().getName().equals("scheduledJob:1"))); + + /////////////// + // repeat it // + /////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(SchedulerTestUtils + .newScheduledJob(ScheduledJobType.PROCESS, Map.of("processName", TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE)) + .withLabel("Test job 2") + .withId(null) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME))); + + input = new RunProcessInput(); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + input.setProcessName(ScheduleAllNewJobsProcess.class.getSimpleName()); + new RunProcessAction().execute(input); + + wrappers = quartzScheduler.queryQuartz(); + assertEquals(2, wrappers.size()); + assertTrue(wrappers.stream().anyMatch(w -> w.jobDetail().getKey().getName().equals("scheduledJob:2"))); + + ///////////////////////////////////////////////////////////////////////////////////// + // make sure quartzScheduler never logged about deleting or re-scheduling anything // + ///////////////////////////////////////////////////////////////////////////////////// + assertThat(quartzSchedulerLog.getCollectedMessages()) + .noneMatch(m -> m.getMessage().toLowerCase().contains("delete")) + .noneMatch(m -> m.getMessage().toLowerCase().contains("re-schedule")); + } + finally + { + QLogger.deactivateCollectingLoggerForClass(SchedulableSQSQueueRunner.class); + } + } + +} \ No newline at end of file