CE-936 - Checkpoint on Quartz scheduler implementation.

- Add QSchedulerMetaData as new type of top-level meta data - Move existing ScheduleManager to be SimpleScheduler, an instance of new QSchedulerInterface - Update QuartzScheduler to implement new QSchedulerInterface, plus: -- support cron schedules -- handle parallel variant jobs -- handle automations & sqs pollers
2025-07-18 21:20:45 +00:00 · 2024-03-12 11:55:14 -05:00
parent dabaafa482
commit 7155180a76
21 changed files with 1770 additions and 299 deletions
--- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java
+++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java
@ -43,7 +43,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
-import com.kingsrook.qqq.backend.core.scheduler.StandardScheduledExecutor;
+import com.kingsrook.qqq.backend.core.scheduler.simple.StandardScheduledExecutor;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.AfterEach;
--- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java
+++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java
@ -28,6 +28,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.function.Supplier;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
@ -69,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMeta
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
@ -367,6 +369,127 @@ class QInstanceValidatorTest extends BaseTest



+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test_validateSchedules()
+   {
+      String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE;
+      Supplier<QScheduleMetaData> baseScheduleMetaData = () -> new QScheduleMetaData()
+         .withSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME);
+
+      ////////////////////////////////////////////////////
+      // do our basic schedule validations on a process //
+      ////////////////////////////////////////////////////
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()),
+         "either repeatMillis or repeatSeconds or cronExpression must be set");
+
+      String validCronString = "* * * * * ?";
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
+            .withRepeatMillis(1)
+            .withCronExpression(validCronString)
+            .withCronTimeZoneId("UTC")),
+         "both a repeat time and cronExpression may not be set");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
+            .withRepeatSeconds(1)
+            .withCronExpression(validCronString)
+            .withCronTimeZoneId("UTC")),
+         "both a repeat time and cronExpression may not be set");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
+            .withRepeatSeconds(1)
+            .withRepeatMillis(1)
+            .withCronExpression(validCronString)
+            .withCronTimeZoneId("UTC")),
+         "both a repeat time and cronExpression may not be set");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
+            .withInitialDelaySeconds(1)
+            .withCronExpression(validCronString)
+            .withCronTimeZoneId("UTC")),
+         "cron schedule may not have an initial delay");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
+            .withInitialDelayMillis(1)
+            .withCronExpression(validCronString)
+            .withCronTimeZoneId("UTC")),
+         "cron schedule may not have an initial delay");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
+            .withInitialDelaySeconds(1)
+            .withInitialDelayMillis(1)
+            .withCronExpression(validCronString)
+            .withCronTimeZoneId("UTC")),
+         "cron schedule may not have an initial delay");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
+            .withCronExpression(validCronString)),
+         "must specify a cronTimeZoneId");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
+            .withCronExpression(validCronString)
+            .withCronTimeZoneId("foobar")),
+         "unrecognized cronTimeZoneId: foobar");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
+            .withCronExpression("* * * * * *")
+            .withCronTimeZoneId("UTC")),
+         "invalid cron expression: Support for specifying both");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
+            .withCronExpression("x")
+            .withCronTimeZoneId("UTC")),
+         "invalid cron expression: Illegal cron expression format");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
+            .withRepeatSeconds(10)
+            .withCronTimeZoneId("UTC")),
+         "non-cron schedule must not specify a cronTimeZoneId");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
+            .withSchedulerName(null)
+            .withCronExpression(validCronString)
+            .withCronTimeZoneId("UTC")),
+         "is missing a scheduler name");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
+            .withSchedulerName("not-a-scheduler")
+            .withCronExpression(validCronString)
+            .withCronTimeZoneId("UTC")),
+         "referencing an unknown scheduler name: not-a-scheduler");
+
+      /////////////////////////////////
+      // validate some success cases //
+      /////////////////////////////////
+      assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withRepeatSeconds(1)));
+      assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withRepeatMillis(1)));
+      assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("UTC")));
+      assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("America/New_York")));
+
+      //////////////////////////////////////////////////////////////////
+      // make sure automation providers get their schedules validated //
+      //////////////////////////////////////////////////////////////////
+      assertValidationFailureReasons((qInstance) -> qInstance.getAutomationProvider(TestUtils.POLLING_AUTOMATION).withSchedule(baseScheduleMetaData.get()
+            .withSchedulerName(null)
+            .withCronExpression(validCronString)
+            .withCronTimeZoneId("UTC")),
+         "is missing a scheduler name");
+
+      /////////////////////////////////////////////////////////////
+      // make sure queue providers get their schedules validated //
+      /////////////////////////////////////////////////////////////
+      assertValidationFailureReasons((qInstance) -> ((SQSQueueProviderMetaData)qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER)).withSchedule(baseScheduleMetaData.get()
+            .withSchedulerName(null)
+            .withCronExpression(validCronString)
+            .withCronTimeZoneId("UTC")),
+         "is missing a scheduler name");
+
+   }
+
+
+
   /*******************************************************************************
    ** Test that a table with no fields fails.
    **
--- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java
+++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java
@ -0,0 +1,63 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.scheduler.quartz;
+
+
+import java.util.Properties;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.quartz.SchedulerException;
+
+
+/*******************************************************************************
+ ** Unit test for QuartzScheduler 
+ *******************************************************************************/
+class QuartzSchedulerTest extends BaseTest
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void beforeEach() throws SchedulerException
+   {
+      Properties quartzProperties = new Properties();
+      quartzProperties.put("", "");
+      quartzProperties.put("org.quartz.scheduler.instanceName", "TestScheduler");
+      quartzProperties.put("org.quartz.threadPool.threadCount", "3");
+      quartzProperties.put("org.quartz.jobStore.class", "org.quartz.simpl.RAMJobStore");
+      QuartzScheduler.initInstance(QContext.getQInstance(), "TestScheduler", quartzProperties, QContext::getQSession);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test()
+   {
+
+   }
+
+}
--- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java
+++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java
@ -1,6 +1,6 @@
 /*
 * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
 * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
 * contact@kingsrook.com
 * https://github.com/Kingsrook/
@ -19,7 +19,7 @@
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

-package com.kingsrook.qqq.backend.core.scheduler;
+package com.kingsrook.qqq.backend.core.scheduler.simple;


 import java.util.List;
@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
@ -48,7 +49,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 /*******************************************************************************
 ** Unit test for ScheduleManager
 *******************************************************************************/
-class ScheduleManagerTest extends BaseTest
+class SimpleSchedulerTest extends BaseTest
 {

   /*******************************************************************************
@ -57,7 +58,7 @@ class ScheduleManagerTest extends BaseTest
   @AfterEach
   void afterEach()
   {
-      ScheduleManager.resetSingleton();
+      SimpleScheduler.resetSingleton();
   }


@ -69,12 +70,13 @@ class ScheduleManagerTest extends BaseTest
   void testStartAndStop()
   {
      QInstance       qInstance       = QContext.getQInstance();
-      ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance);
-      scheduleManager.start();
+      SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance);
+      simpleScheduler.setSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME);
+      simpleScheduler.start();

-      assertThat(scheduleManager.getExecutors()).isNotEmpty();
+      assertThat(simpleScheduler.getExecutors()).isNotEmpty();

-      scheduleManager.stopAsync();
+      simpleScheduler.stopAsync();
   }


@ -101,11 +103,12 @@ class ScheduleManagerTest extends BaseTest

      BasicStep.counter = 0;

-      ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance);
-      scheduleManager.setSessionSupplier(QSession::new);
-      scheduleManager.start();
+      SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance);
+      simpleScheduler.setSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME);
+      simpleScheduler.setSessionSupplier(QSession::new);
+      simpleScheduler.start();
      SleepUtils.sleep(50, TimeUnit.MILLISECONDS);
-      scheduleManager.stopAsync();
+      simpleScheduler.stopAsync();

      System.out.println("Ran: " + BasicStep.counter + " times");
      assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice");
--- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
+++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
@ -92,6 +92,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
+import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple.SimpleSchedulerMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
@ -176,6 +179,8 @@ public class TestUtils
   public static final String SECURITY_KEY_TYPE_STORE_NULL_BEHAVIOR  = "storeNullBehavior";
   public static final String SECURITY_KEY_TYPE_INTERNAL_OR_EXTERNAL = "internalOrExternal";

+   public static final String SIMPLE_SCHEDULER_NAME = "simpleScheduler";
+


   /*******************************************************************************
@ -237,11 +242,23 @@ public class TestUtils
      defineWidgets(qInstance);
      defineApps(qInstance);

+      qInstance.addScheduler(defineSimpleScheduler());
+
      return (qInstance);
   }



+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QSchedulerMetaData defineSimpleScheduler()
+   {
+      return new SimpleSchedulerMetaData().withName(SIMPLE_SCHEDULER_NAME);
+   }
+
+
+
   /*******************************************************************************
    **
    *******************************************************************************/
@ -349,7 +366,10 @@ public class TestUtils
   private static QAutomationProviderMetaData definePollingAutomationProvider()
   {
      return (new PollingAutomationProviderMetaData()
-         .withName(POLLING_AUTOMATION));
+         .withName(POLLING_AUTOMATION)
+         .withSchedule(new QScheduleMetaData()
+            .withSchedulerName(SIMPLE_SCHEDULER_NAME)
+            .withRepeatSeconds(60)));
   }


@ -1313,7 +1333,10 @@ public class TestUtils
         .withAccessKey(accessKey)
         .withSecretKey(secretKey)
         .withRegion(region)
-         .withBaseURL(baseURL));
+         .withBaseURL(baseURL)
+         .withSchedule(new QScheduleMetaData()
+            .withRepeatSeconds(60)
+            .withSchedulerName(SIMPLE_SCHEDULER_NAME)));
   }