diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 96661697..4582bbec 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -25,7 +25,9 @@ package com.kingsrook.qqq.backend.core.instances; import java.io.Serializable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Modifier; +import java.text.ParseException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -33,6 +35,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TimeZone; import java.util.function.Supplier; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; @@ -93,6 +96,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda; +import org.quartz.CronExpression; /******************************************************************************* @@ -342,6 +346,11 @@ public class QInstanceValidator assertCondition(StringUtils.hasContent(sqsQueueProvider.getSecretKey()), "Missing secretKey for SQSQueueProvider: " + name); assertCondition(StringUtils.hasContent(sqsQueueProvider.getBaseURL()), "Missing baseURL for SQSQueueProvider: " + name); assertCondition(StringUtils.hasContent(sqsQueueProvider.getRegion()), "Missing region for SQSQueueProvider: " + name); + + if(assertCondition(sqsQueueProvider.getSchedule() != null, "Missing schedule for SQSQueueProvider: " + name)) + { + validateScheduleMetaData(sqsQueueProvider.getSchedule(), qInstance, "SQSQueueProvider " + name + ", schedule: "); + } } }); } @@ -392,6 +401,11 @@ public class QInstanceValidator { assertCondition(Objects.equals(name, automationProvider.getName()), "Inconsistent naming for automationProvider: " + name + "/" + automationProvider.getName() + "."); assertCondition(automationProvider.getType() != null, "Missing type for automationProvider: " + name); + + if(assertCondition(automationProvider.getSchedule() != null, "Missing schedule for automationProvider: " + name)) + { + validateScheduleMetaData(automationProvider.getSchedule(), qInstance, "automationProvider " + name + ", schedule: "); + } }); } } @@ -1316,7 +1330,7 @@ public class QInstanceValidator if(process.getSchedule() != null) { QScheduleMetaData schedule = process.getSchedule(); - assertCondition(schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null, "Either repeat millis or repeat seconds must be set on schedule in process " + processName); + validateScheduleMetaData(schedule, qInstance, "Process " + processName + ", schedule: "); if(schedule.getVariantBackend() != null) { @@ -1336,6 +1350,50 @@ public class QInstanceValidator + /******************************************************************************* + ** + *******************************************************************************/ + private void validateScheduleMetaData(QScheduleMetaData schedule, QInstance qInstance, String prefix) + { + boolean isRepeat = schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null; + boolean isCron = StringUtils.hasContent(schedule.getCronExpression()); + assertCondition(isRepeat || isCron, prefix + " either repeatMillis or repeatSeconds or cronExpression must be set"); + assertCondition(!(isRepeat && isCron), prefix + " both a repeat time and cronExpression may not be set"); + + if(isCron) + { + boolean hasDelay = schedule.getInitialDelayMillis() != null || schedule.getInitialDelaySeconds() != null; + assertCondition(!hasDelay, prefix + " a cron schedule may not have an initial delay"); + + try + { + CronExpression.validateExpression(schedule.getCronExpression()); + } + catch(ParseException pe) + { + errors.add(prefix + " invalid cron expression: " + pe.getMessage()); + } + + if(assertCondition(StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a cron schedule must specify a cronTimeZoneId")) + { + String[] availableIDs = TimeZone.getAvailableIDs(); + Optional first = Arrays.stream(availableIDs).filter(id -> id.equals(schedule.getCronTimeZoneId())).findFirst(); + assertCondition(first.isPresent(), prefix + " unrecognized cronTimeZoneId: " + schedule.getCronTimeZoneId()); + } + } + else + { + assertCondition(!StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a non-cron schedule must not specify a cronTimeZoneId"); + } + + if(assertCondition(StringUtils.hasContent(schedule.getSchedulerName()), prefix + " is missing a scheduler name")) + { + assertCondition(qInstance.getScheduler(schedule.getSchedulerName()) != null, prefix + " is referencing an unknown scheduler name: " + schedule.getSchedulerName()); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 115fcccc..5a5b4ac3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -53,6 +53,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -90,6 +91,7 @@ public class QInstance private Map widgets = new LinkedHashMap<>(); private Map queueProviders = new LinkedHashMap<>(); private Map queues = new LinkedHashMap<>(); + private Map schedulers = new LinkedHashMap<>(); private Map supplementalMetaData = new LinkedHashMap<>(); @@ -1224,4 +1226,56 @@ public class QInstance metaData.addSelfToInstance(this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addScheduler(QSchedulerMetaData scheduler) + { + String name = scheduler.getName(); + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a scheduler without a name.")); + } + if(this.schedulers.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second scheduler with name: " + name)); + } + this.schedulers.put(name, scheduler); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QSchedulerMetaData getScheduler(String name) + { + return (this.schedulers.get(name)); + } + + + + /******************************************************************************* + ** Getter for schedulers + ** + *******************************************************************************/ + public Map getSchedulers() + { + return schedulers; + } + + + + /******************************************************************************* + ** Setter for schedulers + ** + *******************************************************************************/ + public void setSchedulers(Map schedulers) + { + this.schedulers = schedulers; + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java index ece9019a..c7373f1a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java @@ -22,6 +22,9 @@ package com.kingsrook.qqq.backend.core.model.metadata.scheduleing; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + /******************************************************************************* ** Meta-data to define scheduled actions within QQQ. ** @@ -37,16 +40,29 @@ public class QScheduleMetaData {PARALLEL, SERIAL} + private String schedulerName; private Integer repeatSeconds; private Integer repeatMillis; private Integer initialDelaySeconds; private Integer initialDelayMillis; + private String cronExpression; + private String cronTimeZoneId; + private RunStrategy variantRunStrategy; private String variantBackend; + /******************************************************************************* + ** + *******************************************************************************/ + public boolean isCron() + { + return StringUtils.hasContent(cronExpression); + } + + /******************************************************************************* ** Getter for repeatSeconds @@ -244,4 +260,97 @@ public class QScheduleMetaData return (this); } + + /******************************************************************************* + ** Getter for cronExpression + *******************************************************************************/ + public String getCronExpression() + { + return (this.cronExpression); + } + + + + /******************************************************************************* + ** Setter for cronExpression + *******************************************************************************/ + public void setCronExpression(String cronExpression) + { + this.cronExpression = cronExpression; + } + + + + /******************************************************************************* + ** Fluent setter for cronExpression + *******************************************************************************/ + public QScheduleMetaData withCronExpression(String cronExpression) + { + this.cronExpression = cronExpression; + return (this); + } + + + + /******************************************************************************* + ** Getter for cronTimeZoneId + *******************************************************************************/ + public String getCronTimeZoneId() + { + return (this.cronTimeZoneId); + } + + + + /******************************************************************************* + ** Setter for cronTimeZoneId + *******************************************************************************/ + public void setCronTimeZoneId(String cronTimeZoneId) + { + this.cronTimeZoneId = cronTimeZoneId; + } + + + + /******************************************************************************* + ** Fluent setter for cronTimeZoneId + *******************************************************************************/ + public QScheduleMetaData withCronTimeZoneId(String cronTimeZoneId) + { + this.cronTimeZoneId = cronTimeZoneId; + return (this); + } + + + + /******************************************************************************* + ** Getter for schedulerName + *******************************************************************************/ + public String getSchedulerName() + { + return (this.schedulerName); + } + + + + /******************************************************************************* + ** Setter for schedulerName + *******************************************************************************/ + public void setSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + } + + + + /******************************************************************************* + ** Fluent setter for schedulerName + *******************************************************************************/ + public QScheduleMetaData withSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java new file mode 100644 index 00000000..90fb8f92 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java @@ -0,0 +1,131 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.scheduleing; + + +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class QSchedulerMetaData implements TopLevelMetaDataInterface +{ + private String name; + private String type; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean supportsCronSchedules() + { + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public abstract QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier systemSessionSupplier) throws QException; + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public QSchedulerMetaData withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for type + *******************************************************************************/ + public String getType() + { + return (this.type); + } + + + + /******************************************************************************* + ** Setter for type + *******************************************************************************/ + public void setType(String type) + { + this.type = type; + } + + + + /******************************************************************************* + ** Fluent setter for type + *******************************************************************************/ + public QSchedulerMetaData withType(String type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addSelfToInstance(QInstance qInstance) + { + qInstance.addScheduler(this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java new file mode 100644 index 00000000..12ef0361 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java @@ -0,0 +1,120 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz; + + +import java.util.Properties; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuartzSchedulerMetaData extends QSchedulerMetaData +{ + private static final QLogger LOG = QLogger.getLogger(QuartzSchedulerMetaData.class); + + public static final String TYPE = "quartz"; + + private Properties properties; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public QuartzSchedulerMetaData() + { + setType(TYPE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean supportsCronSchedules() + { + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier systemSessionSupplier) throws QException + { + try + { + QuartzScheduler quartzScheduler = QuartzScheduler.initInstance(qInstance, getName(), getProperties(), systemSessionSupplier); + return (quartzScheduler); + } + catch(Exception e) + { + LOG.error("Error initializing quartz scheduler", e); + throw (new QException("Error initializing quartz scheduler", e)); + } + } + + + + /******************************************************************************* + ** Getter for properties + *******************************************************************************/ + public Properties getProperties() + { + return (this.properties); + } + + + + /******************************************************************************* + ** Setter for properties + *******************************************************************************/ + public void setProperties(Properties properties) + { + this.properties = properties; + } + + + + /******************************************************************************* + ** Fluent setter for properties + *******************************************************************************/ + public QuartzSchedulerMetaData withProperties(Properties properties) + { + this.properties = properties; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java new file mode 100644 index 00000000..92ef2c99 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java @@ -0,0 +1,76 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple; + + +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.simple.SimpleScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SimpleSchedulerMetaData extends QSchedulerMetaData +{ + public static final String TYPE = "simple"; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SimpleSchedulerMetaData() + { + setType(TYPE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean supportsCronSchedules() + { + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier systemSessionSupplier) + { + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); + simpleScheduler.setSessionSupplier(systemSessionSupplier); + simpleScheduler.setSchedulerName(getName()); + return simpleScheduler; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java new file mode 100644 index 00000000..34302e23 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -0,0 +1,278 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.context.CapturedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; +import com.kingsrook.qqq.backend.core.logging.LogPair; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** QQQ service to manage scheduled jobs, using 1 or more Schedulers - implementations + ** of the QSchedulerInterface + *******************************************************************************/ +public class QScheduleManager +{ + private static final QLogger LOG = QLogger.getLogger(QScheduleManager.class); + + private static QScheduleManager qScheduleManager = null; + private final QInstance qInstance; + private final Supplier systemUserSessionSupplier; + + private Map schedulers = new HashMap<>(); + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private QScheduleManager(QInstance qInstance, Supplier systemUserSessionSupplier) + { + this.qInstance = qInstance; + this.systemUserSessionSupplier = systemUserSessionSupplier; + } + + + + /******************************************************************************* + ** Singleton initiator - e.g., must be called to initially initialize the singleton + ** before anyone else calls getInstance (they'll get an error if they call that first). + *******************************************************************************/ + public static QScheduleManager initInstance(QInstance qInstance, Supplier systemUserSessionSupplier) + { + if(qScheduleManager == null) + { + qScheduleManager = new QScheduleManager(qInstance, systemUserSessionSupplier); + } + return (qScheduleManager); + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static QScheduleManager getInstance() + { + if(qScheduleManager == null) + { + throw (new IllegalStateException("QScheduleManager singleton has not been init'ed (call initInstance).")); + } + return (qScheduleManager); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void start() throws QException + { + if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) + { + LOG.info("Not starting ScheduleManager per settings."); + return; + } + + ///////////////////////////////////////////////////////// + // initialize the scheduler(s) we're configured to use // + ///////////////////////////////////////////////////////// + for(QSchedulerMetaData schedulerMetaData : qInstance.getSchedulers().values()) + { + QSchedulerInterface scheduler = schedulerMetaData.initSchedulerInstance(qInstance, systemUserSessionSupplier); + schedulers.put(schedulerMetaData.getName(), scheduler); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // ensure that everything which should be scheduled is scheduled, in the appropriate scheduler // + ///////////////////////////////////////////////////////////////////////////////////////////////// + QContext.withTemporaryContext(new CapturedContext(qInstance, systemUserSessionSupplier.get()), () -> setupSchedules()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupSchedules() + { + ///////////////////////////////////////////////////////// + // let the schedulers know we're starting this process // + ///////////////////////////////////////////////////////// + schedulers.values().forEach(s -> s.startOfSetupSchedules()); + + ////////////////////////////////// + // schedule all queue providers // + ////////////////////////////////// + for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values()) + { + setupQueueProvider(queueProvider); + } + + /////////////////////////////////////// + // schedule all automation providers // + /////////////////////////////////////// + for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values()) + { + setupAutomationProviderPerTable(automationProvider); + } + + ///////////////////////////////////////// + // schedule all processes that need it // + ///////////////////////////////////////// + for(QProcessMetaData process : qInstance.getProcesses().values()) + { + if(process.getSchedule() != null) + { + QScheduleMetaData scheduleMetaData = process.getSchedule(); + if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) + { + /////////////////////////////////////////////// + // if no variants, or variant is serial mode // + /////////////////////////////////////////////// + setupProcess(process, null); + } + else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // if this a "parallel", which for example means we want to have a thread for each backend variant // + // running at the same time, get the variant records and schedule each separately // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); + for(QRecord qRecord : CollectionUtils.nonNullList(SchedulerUtils.getBackendVariantFilteredRecords(process))) + { + try + { + setupProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); + } + catch(Exception e) + { + LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); + } + } + } + else + { + LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided."); + } + } + } + + ///////////////////////////////////////////////////////////// + // todo - read dynamic schedules and schedule those things // + // e.g., user-scheduled processes, reports // + ///////////////////////////////////////////////////////////// + + ////////////////////////////////////////////////////////// + // let the schedulers know we're done with this process // + ////////////////////////////////////////////////////////// + schedulers.values().forEach(s -> s.endOfSetupSchedules()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupProcess(QProcessMetaData process, Map backendVariantData) + { + QSchedulerInterface scheduler = getScheduler(process.getSchedule().getSchedulerName()); + scheduler.setupProcess(process, backendVariantData, SchedulerUtils.allowedToStart(process)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupQueueProvider(QQueueProviderMetaData queueProvider) + { + switch(queueProvider.getType()) + { + case SQS: + setupSqsProvider((SQSQueueProviderMetaData) queueProvider); + break; + default: + throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupSqsProvider(SQSQueueProviderMetaData queueProvider) + { + QSchedulerInterface scheduler = getScheduler(queueProvider.getSchedule().getSchedulerName()); + scheduler.setupSqsProvider(queueProvider, SchedulerUtils.allowedToStart(queueProvider)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider) + { + QSchedulerInterface scheduler = getScheduler(automationProvider.getSchedule().getSchedulerName()); + scheduler.setupAutomationProviderPerTable(automationProvider, SchedulerUtils.allowedToStart(automationProvider)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QSchedulerInterface getScheduler(String schedulerName) + { + QSchedulerInterface scheduler = schedulers.get(schedulerName); + if(scheduler == null) + { + throw new NotImplementedException("default scheduler..."); + } + + return (scheduler); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java new file mode 100644 index 00000000..62404fb0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java @@ -0,0 +1,84 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.io.Serializable; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface QSchedulerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + void setupSqsProvider(SQSQueueProviderMetaData queueProvider, boolean allowedToStart); + + /******************************************************************************* + ** + *******************************************************************************/ + void stopAsync(); + + /******************************************************************************* + ** + *******************************************************************************/ + void stop(); + + /******************************************************************************* + ** + *******************************************************************************/ + void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStart); + + /******************************************************************************* + ** + *******************************************************************************/ + void setupProcess(QProcessMetaData process, Map backendVariantData, boolean allowedToStart); + + /******************************************************************************* + ** + *******************************************************************************/ + void start(); + + /******************************************************************************* + ** let the scheduler know when the schedule manager is at the start of setting up schedules. + *******************************************************************************/ + default void startOfSetupSchedules() + { + + } + + /******************************************************************************* + ** let the scheduler know when the schedule manager is at the end of setting up schedules. + *******************************************************************************/ + default void endOfSetupSchedules() + { + + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java index 76f86394..e44c124d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -48,7 +49,7 @@ import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; /******************************************************************************* - ** + ** Utility methods used by various schedulers. *******************************************************************************/ public class SchedulerUtils { @@ -56,6 +57,16 @@ public class SchedulerUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static boolean allowedToStart(TopLevelMetaDataInterface metaDataObject) + { + return (allowedToStart(metaDataObject.getName())); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java index a5bb27f3..3cfde45f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java @@ -27,6 +27,7 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; @@ -57,16 +58,31 @@ public class QuartzRunProcessJob implements Job JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); String processName = jobDataMap.getString("processName"); - /////////////////////////////////// - // todo - variants from job data // - /////////////////////////////////// + /////////////////////////////////////// + // get the process from the instance // + /////////////////////////////////////// + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + QProcessMetaData process = qInstance.getProcess(processName); + if(process == null) + { + LOG.warn("Could not find scheduled process in QInstance", logPair("processName", processName)); + return; + } + + /////////////////////////////////////////////// + // if the job has variant data, get it ready // + /////////////////////////////////////////////// Map backendVariantData = null; + if(jobExecutionContext.getMergedJobDataMap().containsKey("backendVariantData")) + { + backendVariantData = (Map) jobExecutionContext.getMergedJobDataMap().get("backendVariantData"); + } + ///////////// + // run it. // + ///////////// LOG.debug("Running quartz process", logPair("processName", processName)); - - QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); SchedulerUtils.runProcess(qInstance, QuartzScheduler.getInstance().getSessionSupplier(), qInstance.getProcess(processName), backendVariantData); - } finally { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java index 5a8385f6..da9fe9ca 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java @@ -23,20 +23,38 @@ package com.kingsrook.qqq.backend.core.scheduler.quartz; import java.io.Serializable; +import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.Date; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.TimeZone; import java.util.function.Supplier; -import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.AnyKey; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; +import org.quartz.CronExpression; +import org.quartz.CronScheduleBuilder; +import org.quartz.Job; import org.quartz.JobBuilder; import org.quartz.JobDetail; import org.quartz.JobKey; +import org.quartz.ScheduleBuilder; import org.quartz.Scheduler; import org.quartz.SchedulerException; import org.quartz.SimpleScheduleBuilder; @@ -49,41 +67,60 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* - ** + ** Singleton to provide access between QQQ and the quartz Scheduler system. *******************************************************************************/ -public class QuartzScheduler +public class QuartzScheduler implements QSchedulerInterface { private static final QLogger LOG = QLogger.getLogger(QuartzScheduler.class); private static QuartzScheduler quartzScheduler = null; private final QInstance qInstance; + private String schedulerName; + private Properties quartzProperties; private Supplier sessionSupplier; private Scheduler scheduler; + private Memoization> jobGroupNamesMemoization = new Memoization>() + .withTimeout(Duration.of(5, ChronoUnit.SECONDS)); + + private Memoization> jobKeyNamesMemoization = new Memoization>() + .withTimeout(Duration.of(5, ChronoUnit.SECONDS)); + /******************************************************************************* ** Constructor ** *******************************************************************************/ - private QuartzScheduler(QInstance qInstance, Supplier sessionSupplier) + private QuartzScheduler(QInstance qInstance, String schedulerName, Properties quartzProperties, Supplier sessionSupplier) { this.qInstance = qInstance; + this.schedulerName = schedulerName; + this.quartzProperties = quartzProperties; this.sessionSupplier = sessionSupplier; } /******************************************************************************* - ** Singleton initiator... + ** Singleton initiator - e.g., must be called to initially initialize the singleton + ** before anyone else calls getInstance (they'll get an error if they call that first). *******************************************************************************/ - public static QuartzScheduler initInstance(QInstance qInstance, Supplier sessionSupplier) + public static QuartzScheduler initInstance(QInstance qInstance, String schedulerName, Properties quartzProperties, Supplier sessionSupplier) throws SchedulerException { if(quartzScheduler == null) { - quartzScheduler = new QuartzScheduler(qInstance, sessionSupplier); + quartzScheduler = new QuartzScheduler(qInstance, schedulerName, quartzProperties, sessionSupplier); + + /////////////////////////////////////////////////////////// + // Grab the Scheduler instance from the Factory // + // initialize it with the properties we took in as input // + /////////////////////////////////////////////////////////// + StdSchedulerFactory schedulerFactory = new StdSchedulerFactory(); + schedulerFactory.initialize(quartzProperties); + quartzScheduler.scheduler = schedulerFactory.getScheduler(); } return (quartzScheduler); } @@ -97,7 +134,7 @@ public class QuartzScheduler { if(quartzScheduler == null) { - throw (new IllegalStateException("QuartzScheduler singleton has not been init'ed.")); + throw (new IllegalStateException("QuartzScheduler singleton has not been init'ed (call initInstance).")); } return (quartzScheduler); } @@ -111,30 +148,6 @@ public class QuartzScheduler { try { - // Properties properties = new Properties(); - // properties.put(""); - - ////////////////////////////////////////////////// - // Grab the Scheduler instance from the Factory // - ////////////////////////////////////////////////// - StdSchedulerFactory schedulerFactory = new StdSchedulerFactory(); - // schedulerFactory.initialize(properties); - this.scheduler = schedulerFactory.getScheduler(); - - //////////////////////////////////////// - // todo - do we get our own property? // - //////////////////////////////////////// - if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) - { - LOG.info("Not starting QuartzScheduler per settings."); - return; - } - - ///////////////////////////////////////////// - // make sure all of our jobs are scheduled // - ///////////////////////////////////////////// - scheduleAllJobs(); - ////////////////////// // and start it off // ////////////////////// @@ -151,20 +164,166 @@ public class QuartzScheduler /******************************************************************************* ** *******************************************************************************/ - private void scheduleAllJobs() + @Override + public void stop() { - for(QProcessMetaData process : qInstance.getProcesses().values()) + try { - if(process.getSchedule() != null && SchedulerUtils.allowedToStart(process.getName())) + scheduler.shutdown(true); + } + catch(SchedulerException e) + { + LOG.error("Error shutting down (stopping) quartz scheduler", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stopAsync() + { + try + { + scheduler.shutdown(false); + } + catch(SchedulerException e) + { + LOG.error("Error shutting down (stopping) quartz scheduler", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setupProcess(QProcessMetaData process, Map backendVariantData, boolean allowedToStart) + { + ///////////////////////// + // set up job data map // + ///////////////////////// + Map jobData = new HashMap<>(); + jobData.put("processName", process.getName()); + + if(backendVariantData != null) + { + jobData.put("backendVariantData", backendVariantData); + } + + scheduleJob(process.getName(), "processes", QuartzRunProcessJob.class, jobData, process.getSchedule(), allowedToStart); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean scheduleJob(String jobName, String groupName, Class jobClass, Map jobData, QScheduleMetaData scheduleMetaData, boolean allowedToStart) + { + try + { + ///////////////////////// + // Define job instance // + ///////////////////////// + JobKey jobKey = new JobKey(jobName, groupName); + JobDetail jobDetail = JobBuilder.newJob(jobClass) + .withIdentity(jobKey) + .storeDurably() + .requestRecovery() + .build(); + + jobDetail.getJobDataMap().putAll(jobData); + + ///////////////////////////////////////////////////////// + // map the qqq schedule meta data to a quartz schedule // + ///////////////////////////////////////////////////////// + ScheduleBuilder scheduleBuilder; + if(scheduleMetaData.isCron()) { - if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) - { - scheduleProcess(process, null); - } - else - { - LOG.error("Not yet know how to schedule parallel variant jobs"); - } + CronExpression cronExpression = new CronExpression(scheduleMetaData.getCronExpression()); + cronExpression.setTimeZone(TimeZone.getTimeZone(scheduleMetaData.getCronTimeZoneId())); + scheduleBuilder = CronScheduleBuilder.cronSchedule(cronExpression); + } + else + { + long intervalMillis = Objects.requireNonNullElse(scheduleMetaData.getRepeatMillis(), scheduleMetaData.getRepeatSeconds() * 1000); + scheduleBuilder = SimpleScheduleBuilder.simpleSchedule() + .withIntervalInMilliseconds(intervalMillis) + .repeatForever(); + } + + Date startAt = new Date(); + if(scheduleMetaData.getInitialDelayMillis() != null) + { + startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelayMillis()); + } + else if(scheduleMetaData.getInitialDelaySeconds() != null) + { + startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelaySeconds() * 1000); + } + + /////////////////////////////////////// + // Define a Trigger for the schedule // + /////////////////////////////////////// + Trigger trigger = TriggerBuilder.newTrigger() + .withIdentity(new TriggerKey(jobName, groupName)) + .forJob(jobKey) + .withSchedule(scheduleBuilder) + .startAt(startAt) + .build(); + + /////////////////////////////////////// + // Schedule the job with the trigger // + /////////////////////////////////////// + addOrReplaceJobAndTrigger(jobKey, jobDetail, trigger); + + ////////////////////////////////////////////////////////// + // either pause or resume, based on if allowed to start // + ////////////////////////////////////////////////////////// + if(!allowedToStart) + { + pauseJob(jobKey.getName(), jobKey.getGroup()); + } + else + { + resumeJob(jobKey.getName(), jobKey.getGroup()); + } + + return (true); + } + catch(Exception e) + { + LOG.warn("Error scheduling job", e, logPair("name", jobName), logPair("group", groupName)); + return (false); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setupSqsProvider(SQSQueueProviderMetaData sqsQueueProvider, boolean allowedToStartProvider) + { + for(QQueueMetaData queue : qInstance.getQueues().values()) + { + boolean allowedToStart = allowedToStartProvider && SchedulerUtils.allowedToStart(queue.getName()); + + if(sqsQueueProvider.getName().equals(queue.getProviderName())) + { + ///////////////////////// + // set up job data map // + ///////////////////////// + Map jobData = new HashMap<>(); + jobData.put("queueProviderName", sqsQueueProvider.getName()); + jobData.put("queueName", queue.getName()); + + scheduleJob(queue.getName(), "sqsQueue", QuartzSqsPollerJob.class, jobData, sqsQueueProvider.getSchedule(), allowedToStart); } } } @@ -174,85 +333,75 @@ public class QuartzScheduler /******************************************************************************* ** *******************************************************************************/ - private void scheduleProcess(QProcessMetaData process, Map backendVariantData) + @Override + public void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStartProvider) { - try + /////////////////////////////////////////////////////////////////////////////////// + // ask the PollingAutomationPerTableRunner how many threads of itself need setup // + // then start a scheduled executor foreach one // + /////////////////////////////////////////////////////////////////////////////////// + List tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName()); + for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions) { - QScheduleMetaData scheduleMetaData = process.getSchedule(); - long intervalMillis = Objects.requireNonNullElse(scheduleMetaData.getRepeatMillis(), scheduleMetaData.getRepeatSeconds() * 1000); - - Date startAt = new Date(); - if(scheduleMetaData.getInitialDelayMillis() != null) - { - startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelayMillis()); - } - if(scheduleMetaData.getInitialDelaySeconds() != null) - { - startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelaySeconds() * 1000); - } + boolean allowedToStart = allowedToStartProvider && SchedulerUtils.allowedToStart(tableAction.tableName()); ///////////////////////// - // Define job instance // + // set up job data map // ///////////////////////// - JobKey jobKey = new JobKey(process.getName(), "processes"); - JobDetail jobDetail = JobBuilder.newJob(QuartzRunProcessJob.class) - .withIdentity(jobKey) - .storeDurably() - .requestRecovery() - .build(); + Map jobData = new HashMap<>(); + jobData.put("automationProviderName", automationProvider.getName()); + jobData.put("tableName", tableAction.tableName()); + jobData.put("automationStatus", tableAction.status().toString()); - jobDetail.getJobDataMap().put("processName", process.getName()); - - /////////////////////////////////////// - // Define a Trigger for the schedule // - /////////////////////////////////////// - Trigger trigger = TriggerBuilder.newTrigger() - .withIdentity(new TriggerKey(process.getName(), "processes")) - .forJob(jobKey) - .withSchedule(SimpleScheduleBuilder.simpleSchedule() - .withIntervalInMilliseconds(intervalMillis) - .repeatForever()) - .startAt(startAt) - .build(); - - /////////////////////////////////////// - // Schedule the job with the trigger // - /////////////////////////////////////// - boolean isJobAlreadyScheduled = isJobAlreadyScheduled(jobKey); - if(isJobAlreadyScheduled) - { - this.scheduler.addJob(jobDetail, true); - this.scheduler.rescheduleJob(trigger.getKey(), trigger); - LOG.info("Re-scheduled process: " + process.getName()); - } - else - { - this.scheduler.scheduleJob(jobDetail, trigger); - LOG.info("Scheduled new process: " + process.getName()); - } - - } - catch(Exception e) - { - LOG.warn("Error scheduling process", e, logPair("processName", process.getName())); + scheduleJob(tableAction.tableName() + "." + tableAction.status(), "tableAutomations", QuartzTableAutomationsJob.class, jobData, automationProvider.getSchedule(), allowedToStart); } } /******************************************************************************* - ** todo - probably rewrite this to not re-query quartz each time + ** + *******************************************************************************/ + private void addOrReplaceJobAndTrigger(JobKey jobKey, JobDetail jobDetail, Trigger trigger) throws SchedulerException + { + boolean isJobAlreadyScheduled = isJobAlreadyScheduled(jobKey); + if(isJobAlreadyScheduled) + { + this.scheduler.addJob(jobDetail, true); + this.scheduler.rescheduleJob(trigger.getKey(), trigger); + LOG.info("Re-scheduled job: " + jobKey); + } + else + { + this.scheduler.scheduleJob(jobDetail, trigger); + LOG.info("Scheduled new job: " + jobKey); + } + } + + + + /******************************************************************************* + ** *******************************************************************************/ private boolean isJobAlreadyScheduled(JobKey jobKey) throws SchedulerException { - for(String group : scheduler.getJobGroupNames()) + Optional> jobGroupNames = jobGroupNamesMemoization.getResult(AnyKey.getInstance(), (x) -> scheduler.getJobGroupNames()); + if(jobGroupNames.isEmpty()) { - for(JobKey testJobKey : scheduler.getJobKeys(GroupMatcher.groupEquals(group))) + throw (new SchedulerException("Error getting job group names")); + } + + for(String group : jobGroupNames.get()) + { + Optional> jobKeys = jobKeyNamesMemoization.getResult(group, (x) -> scheduler.getJobKeys(GroupMatcher.groupEquals(group))); + if(jobKeys.isEmpty()) { - if(testJobKey.equals(jobKey)) - { - return (true); - } + throw (new SchedulerException("Error getting job keys")); + } + + if(jobKeys.get().contains(jobKey)) + { + return (true); } } @@ -261,45 +410,33 @@ public class QuartzScheduler - /* - private void todo() throws SchedulerException + /******************************************************************************* + ** + *******************************************************************************/ + private boolean deleteJob(JobKey jobKey) { - // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/ListJobs.html - // Listing all Jobs in the scheduler - for(String group : scheduler.getJobGroupNames()) + try { - for(JobKey jobKey : scheduler.getJobKeys(GroupMatcher.groupEquals(group))) + ///////////////////////////////////////////////////////////////////////////////////////////// + // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UnscheduleJob.html // + // Deleting a Job and Unscheduling All of Its Triggers // + ///////////////////////////////////////////////////////////////////////////////////////////// + if(isJobAlreadyScheduled(jobKey)) { - System.out.println("Found job identified by: " + jobKey); + return scheduler.deleteJob(jobKey); } + + ///////////////////////////////////////// + // return true to indicate, we're good // + ///////////////////////////////////////// + return (true); + } + catch(Exception e) + { + LOG.warn("Error deleting job", e, logPair("jobKey", jobKey)); + return false; } - - // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UpdateJob.html - // Update an existing job - // Add the new job to the scheduler, instructing it to "replace" - // the existing job with the given name and group (if any) - JobDetail jobDetail = JobBuilder.newJob(QuartzRunProcessJob.class) - .withIdentity("job1", "group1") - .build(); - // store, and set overwrite flag to 'true' - scheduler.addJob(jobDetail, true); - - // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UpdateTrigger.html - // Define a new Trigger - Trigger trigger = TriggerBuilder.newTrigger() - .withIdentity("newTrigger", "group1") - .startNow() - .build(); - - // tell the scheduler to remove the old trigger with the given key, and put the new one in its place - scheduler.rescheduleJob(new TriggerKey("oldTrigger", "group1"), trigger); - - // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UnscheduleJob.html - // Deleting a Job and Unscheduling All of Its Triggers - scheduler.deleteJob(new JobKey("job1", "group1")); - } - */ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java new file mode 100644 index 00000000..7270d7fa --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java @@ -0,0 +1,79 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +@DisallowConcurrentExecution +public class QuartzSqsPollerJob implements Job +{ + private static final QLogger LOG = QLogger.getLogger(QuartzSqsPollerJob.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException + { + try + { + JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); + String queueProviderName = jobDataMap.getString("queueProviderName"); + String queueName = jobDataMap.getString("queueName"); + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + + SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); + sqsQueuePoller.setQueueProviderMetaData((SQSQueueProviderMetaData) qInstance.getQueueProvider(queueProviderName)); + sqsQueuePoller.setQueueMetaData(qInstance.getQueue(queueName)); + sqsQueuePoller.setQInstance(qInstance); + sqsQueuePoller.setSessionSupplier(QuartzScheduler.getInstance().getSessionSupplier()); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running quartz SQS Poller", logPair("queueName", queueName)); + sqsQueuePoller.run(); + } + finally + { + QContext.clear(); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java new file mode 100644 index 00000000..4f4f244b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +@DisallowConcurrentExecution +public class QuartzTableAutomationsJob implements Job +{ + private static final QLogger LOG = QLogger.getLogger(QuartzTableAutomationsJob.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException + { + try + { + JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); + String tableName = jobDataMap.getString("tableName"); + String automationProviderName = jobDataMap.getString("automationProviderName"); + AutomationStatus automationStatus = AutomationStatus.valueOf(jobDataMap.getString("automationStatus")); + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + + PollingAutomationPerTableRunner.TableActionsInterface tableAction = new PollingAutomationPerTableRunner.TableActions(tableName, automationStatus); + PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProviderName, QuartzScheduler.getInstance().getSessionSupplier(), tableAction); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running Table Automations", logPair("tableName", tableName), logPair("automationStatus", automationStatus)); + runner.run(); + } + finally + { + QContext.clear(); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDetailsPostQueryCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDetailsPostQueryCustomizer.java new file mode 100644 index 00000000..3c08d4ba --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDetailsPostQueryCustomizer.java @@ -0,0 +1,71 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.apache.commons.lang3.SerializationUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuartzJobDetailsPostQueryCustomizer extends AbstractPostQueryCustomizer +{ + private static final QLogger LOG = QLogger.getLogger(QuartzJobDetailsPostQueryCustomizer.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List apply(List records) + { + for(QRecord record : records) + { + if(record.getValue("jobData") != null) + { + try + { + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // this field has a blob of essentially a serialized map - so, deserialize that, then convert to JSON // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + byte[] value = record.getValueByteArray("jobData"); + Object deserialize = SerializationUtils.deserialize(value); + String json = JsonUtils.toJson(deserialize); + record.setValue("jobData", json); + } + catch(Exception e) + { + LOG.info("Error deserializing quartz job data", e); + } + } + } + return (records); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java similarity index 63% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java index cb7e0388..ed215833 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.scheduler; +package com.kingsrook.qqq.backend.core.scheduler.simple; import java.io.Serializable; @@ -30,22 +30,16 @@ import java.util.Objects; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; -import com.kingsrook.qqq.backend.core.logging.LogPair; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; /******************************************************************************* @@ -58,12 +52,13 @@ import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; ** ** All of these jobs run using a "system session" - as defined by the sessionSupplier. *******************************************************************************/ -public class ScheduleManager +public class SimpleScheduler implements QSchedulerInterface { - private static final QLogger LOG = QLogger.getLogger(ScheduleManager.class); + private static final QLogger LOG = QLogger.getLogger(SimpleScheduler.class); - private static ScheduleManager scheduleManager = null; + private static SimpleScheduler simpleScheduler = null; private final QInstance qInstance; + private String schedulerName; protected Supplier sessionSupplier; @@ -79,7 +74,7 @@ public class ScheduleManager /******************************************************************************* ** Singleton constructor *******************************************************************************/ - private ScheduleManager(QInstance qInstance) + private SimpleScheduler(QInstance qInstance) { this.qInstance = qInstance; } @@ -89,13 +84,13 @@ public class ScheduleManager /******************************************************************************* ** Singleton accessor *******************************************************************************/ - public static ScheduleManager getInstance(QInstance qInstance) + public static SimpleScheduler getInstance(QInstance qInstance) { - if(scheduleManager == null) + if(simpleScheduler == null) { - scheduleManager = new ScheduleManager(qInstance); + simpleScheduler = new SimpleScheduler(qInstance); } - return (scheduleManager); + return (simpleScheduler); } @@ -103,88 +98,56 @@ public class ScheduleManager /******************************************************************************* ** *******************************************************************************/ + @Override public void start() { - if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) + for(StandardScheduledExecutor executor : executors) + { + executor.start(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stopAsync() + { + for(StandardScheduledExecutor scheduledExecutor : executors) + { + scheduledExecutor.stopAsync(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stop() + { + for(StandardScheduledExecutor scheduledExecutor : executors) + { + scheduledExecutor.stop(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStartProvider) + { + if(!allowedToStartProvider) { - LOG.info("Not starting ScheduleManager per settings."); return; } - boolean needToClearContext = false; - try - { - if(QContext.getQInstance() == null) - { - needToClearContext = true; - QContext.init(qInstance, sessionSupplier.get()); - } - - for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values()) - { - startQueueProvider(queueProvider); - } - - for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values()) - { - startAutomationProviderPerTable(automationProvider); - } - - for(QProcessMetaData process : qInstance.getProcesses().values()) - { - if(process.getSchedule() != null && SchedulerUtils.allowedToStart(process.getName())) - { - QScheduleMetaData scheduleMetaData = process.getSchedule(); - if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) - { - /////////////////////////////////////////////// - // if no variants, or variant is serial mode // - /////////////////////////////////////////////// - startProcess(process, null); - } - else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) - { - ///////////////////////////////////////////////////////////////////////////////////////////////////// - // if this a "parallel", which for example means we want to have a thread for each backend variant // - // running at the same time, get the variant records and schedule each separately // - ///////////////////////////////////////////////////////////////////////////////////////////////////// - QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); - for(QRecord qRecord : CollectionUtils.nonNullList(SchedulerUtils.getBackendVariantFilteredRecords(process))) - { - try - { - startProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); - } - catch(Exception e) - { - LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); - } - } - } - else - { - LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided."); - } - } - } - } - finally - { - if(needToClearContext) - { - QContext.clear(); - } - } - } - - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void startAutomationProviderPerTable(QAutomationProviderMetaData automationProvider) - { /////////////////////////////////////////////////////////////////////////////////// // ask the PollingAutomationPerTableRunner how many threads of itself need setup // // then start a scheduled executor foreach one // @@ -201,11 +164,6 @@ public class ScheduleManager executor.setName(runner.getName()); setScheduleInExecutor(schedule, executor); - if(!executor.start()) - { - LOG.warn("executor.start return false for: " + executor.getName()); - } - executors.add(executor); } } @@ -216,28 +174,14 @@ public class ScheduleManager /******************************************************************************* ** *******************************************************************************/ - private void startQueueProvider(QQueueProviderMetaData queueProvider) + @Override + public void setupSqsProvider(SQSQueueProviderMetaData queueProvider, boolean allowedToStartProvider) { - if(SchedulerUtils.allowedToStart(queueProvider.getName())) + if(!allowedToStartProvider) { - switch(queueProvider.getType()) - { - case SQS: - startSqsProvider((SQSQueueProviderMetaData) queueProvider); - break; - default: - throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType()); - } + return; } - } - - - /******************************************************************************* - ** - *******************************************************************************/ - private void startSqsProvider(SQSQueueProviderMetaData queueProvider) - { QInstance scheduleManagerQueueInstance = qInstance; Supplier scheduleManagerSessionSupplier = sessionSupplier; @@ -259,11 +203,6 @@ public class ScheduleManager executor.setName(queue.getName()); setScheduleInExecutor(schedule, executor); - if(!executor.start()) - { - LOG.warn("executor.start return false for: " + executor.getName()); - } - executors.add(executor); } } @@ -274,8 +213,14 @@ public class ScheduleManager /******************************************************************************* ** *******************************************************************************/ - private void startProcess(QProcessMetaData process, Map backendVariantData) + @Override + public void setupProcess(QProcessMetaData process, Map backendVariantData, boolean allowedToStart) { + if(!allowedToStart) + { + return; + } + Runnable runProcess = () -> { SchedulerUtils.runProcess(qInstance, sessionSupplier, process, backendVariantData); @@ -284,11 +229,6 @@ public class ScheduleManager StandardScheduledExecutor executor = new StandardScheduledExecutor(runProcess); executor.setName("process:" + process.getName()); setScheduleInExecutor(process.getSchedule(), executor); - if(!executor.start()) - { - LOG.warn("executor.start return false for: " + executor.getName()); - } - executors.add(executor); } @@ -363,22 +303,40 @@ public class ScheduleManager /******************************************************************************* ** *******************************************************************************/ - public void stopAsync() + static void resetSingleton() { - for(StandardScheduledExecutor scheduledExecutor : executors) - { - scheduledExecutor.stopAsync(); - } + simpleScheduler = null; } /******************************************************************************* - ** + ** Getter for schedulerName *******************************************************************************/ - static void resetSingleton() + public String getSchedulerName() { - scheduleManager = null; + return (this.schedulerName); + } + + + + /******************************************************************************* + ** Setter for schedulerName + *******************************************************************************/ + public void setSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + } + + + + /******************************************************************************* + ** Fluent setter for schedulerName + *******************************************************************************/ + public SimpleScheduler withSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + return (this); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java similarity index 98% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java index 3910e65f..abaea628 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.scheduler; +package com.kingsrook.qqq.backend.core.scheduler.simple; import java.util.concurrent.Executors; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java index 57e0055d..e08b4ac1 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java @@ -43,7 +43,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; -import com.kingsrook.qqq.backend.core.scheduler.StandardScheduledExecutor; +import com.kingsrook.qqq.backend.core.scheduler.simple.StandardScheduledExecutor; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index b296ff25..251d42f7 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; @@ -69,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMeta import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; @@ -367,6 +369,127 @@ class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test_validateSchedules() + { + String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE; + Supplier baseScheduleMetaData = () -> new QScheduleMetaData() + .withSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME); + + //////////////////////////////////////////////////// + // do our basic schedule validations on a process // + //////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()), + "either repeatMillis or repeatSeconds or cronExpression must be set"); + + String validCronString = "* * * * * ?"; + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "both a repeat time and cronExpression may not be set"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatSeconds(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "both a repeat time and cronExpression may not be set"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatSeconds(1) + .withRepeatMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "both a repeat time and cronExpression may not be set"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withInitialDelaySeconds(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "cron schedule may not have an initial delay"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withInitialDelayMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "cron schedule may not have an initial delay"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withInitialDelaySeconds(1) + .withInitialDelayMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "cron schedule may not have an initial delay"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression(validCronString)), + "must specify a cronTimeZoneId"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression(validCronString) + .withCronTimeZoneId("foobar")), + "unrecognized cronTimeZoneId: foobar"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression("* * * * * *") + .withCronTimeZoneId("UTC")), + "invalid cron expression: Support for specifying both"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression("x") + .withCronTimeZoneId("UTC")), + "invalid cron expression: Illegal cron expression format"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatSeconds(10) + .withCronTimeZoneId("UTC")), + "non-cron schedule must not specify a cronTimeZoneId"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withSchedulerName(null) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "is missing a scheduler name"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withSchedulerName("not-a-scheduler") + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "referencing an unknown scheduler name: not-a-scheduler"); + + ///////////////////////////////// + // validate some success cases // + ///////////////////////////////// + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withRepeatSeconds(1))); + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withRepeatMillis(1))); + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("UTC"))); + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("America/New_York"))); + + ////////////////////////////////////////////////////////////////// + // make sure automation providers get their schedules validated // + ////////////////////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> qInstance.getAutomationProvider(TestUtils.POLLING_AUTOMATION).withSchedule(baseScheduleMetaData.get() + .withSchedulerName(null) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "is missing a scheduler name"); + + ///////////////////////////////////////////////////////////// + // make sure queue providers get their schedules validated // + ///////////////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> ((SQSQueueProviderMetaData)qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER)).withSchedule(baseScheduleMetaData.get() + .withSchedulerName(null) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "is missing a scheduler name"); + + } + + + /******************************************************************************* ** Test that a table with no fields fails. ** diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java new file mode 100644 index 00000000..42fc18c5 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java @@ -0,0 +1,63 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import java.util.Properties; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.quartz.SchedulerException; + + +/******************************************************************************* + ** Unit test for QuartzScheduler + *******************************************************************************/ +class QuartzSchedulerTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws SchedulerException + { + Properties quartzProperties = new Properties(); + quartzProperties.put("", ""); + quartzProperties.put("org.quartz.scheduler.instanceName", "TestScheduler"); + quartzProperties.put("org.quartz.threadPool.threadCount", "3"); + quartzProperties.put("org.quartz.jobStore.class", "org.quartz.simpl.RAMJobStore"); + QuartzScheduler.initInstance(QContext.getQInstance(), "TestScheduler", quartzProperties, QContext::getQSession); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java similarity index 85% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java index b164376f..8f1aa280 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.scheduler; +package com.kingsrook.qqq.backend.core.scheduler.simple; import java.util.List; @@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -48,7 +49,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* ** Unit test for ScheduleManager *******************************************************************************/ -class ScheduleManagerTest extends BaseTest +class SimpleSchedulerTest extends BaseTest { /******************************************************************************* @@ -57,7 +58,7 @@ class ScheduleManagerTest extends BaseTest @AfterEach void afterEach() { - ScheduleManager.resetSingleton(); + SimpleScheduler.resetSingleton(); } @@ -69,12 +70,13 @@ class ScheduleManagerTest extends BaseTest void testStartAndStop() { QInstance qInstance = QContext.getQInstance(); - ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance); - scheduleManager.start(); + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); + simpleScheduler.setSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME); + simpleScheduler.start(); - assertThat(scheduleManager.getExecutors()).isNotEmpty(); + assertThat(simpleScheduler.getExecutors()).isNotEmpty(); - scheduleManager.stopAsync(); + simpleScheduler.stopAsync(); } @@ -101,11 +103,12 @@ class ScheduleManagerTest extends BaseTest BasicStep.counter = 0; - ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance); - scheduleManager.setSessionSupplier(QSession::new); - scheduleManager.start(); + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); + simpleScheduler.setSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME); + simpleScheduler.setSessionSupplier(QSession::new); + simpleScheduler.start(); SleepUtils.sleep(50, TimeUnit.MILLISECONDS); - scheduleManager.stopAsync(); + simpleScheduler.stopAsync(); System.out.println("Ran: " + BasicStep.counter + " times"); assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 2e3a937e..fde45605 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -92,6 +92,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple.SimpleSchedulerMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; @@ -176,6 +179,8 @@ public class TestUtils public static final String SECURITY_KEY_TYPE_STORE_NULL_BEHAVIOR = "storeNullBehavior"; public static final String SECURITY_KEY_TYPE_INTERNAL_OR_EXTERNAL = "internalOrExternal"; + public static final String SIMPLE_SCHEDULER_NAME = "simpleScheduler"; + /******************************************************************************* @@ -237,11 +242,23 @@ public class TestUtils defineWidgets(qInstance); defineApps(qInstance); + qInstance.addScheduler(defineSimpleScheduler()); + return (qInstance); } + /******************************************************************************* + ** + *******************************************************************************/ + private static QSchedulerMetaData defineSimpleScheduler() + { + return new SimpleSchedulerMetaData().withName(SIMPLE_SCHEDULER_NAME); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -349,7 +366,10 @@ public class TestUtils private static QAutomationProviderMetaData definePollingAutomationProvider() { return (new PollingAutomationProviderMetaData() - .withName(POLLING_AUTOMATION)); + .withName(POLLING_AUTOMATION) + .withSchedule(new QScheduleMetaData() + .withSchedulerName(SIMPLE_SCHEDULER_NAME) + .withRepeatSeconds(60))); } @@ -1313,7 +1333,10 @@ public class TestUtils .withAccessKey(accessKey) .withSecretKey(secretKey) .withRegion(region) - .withBaseURL(baseURL)); + .withBaseURL(baseURL) + .withSchedule(new QScheduleMetaData() + .withRepeatSeconds(60) + .withSchedulerName(SIMPLE_SCHEDULER_NAME))); }