QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-936 - Checkpoint on Quartz scheduler implementation.

Browse Source 
- Add QSchedulerMetaData as new type of top-level meta data
- Move existing ScheduleManager to be SimpleScheduler, an instance of new QSchedulerInterface
- Update QuartzScheduler to implement new QSchedulerInterface, plus:
-- support cron schedules
-- handle parallel variant jobs
-- handle automations & sqs pollers
This commit is contained in:
darin.kelkhoff
2024-03-12 11:55:14 -05:00
parent dabaafa482
commit 7155180a76
21 changed files with 1770 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
View File

@ -25,7 +25,9 @@ package com.kingsrook.qqq.backend.core.instances;
import java.io.Serializable;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Modifier;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@ -33,6 +35,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TimeZone;
import java.util.function.Supplier;
import java.util.stream.Stream;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
@ -93,6 +96,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda;
import org.quartz.CronExpression;
/*******************************************************************************
@ -342,6 +346,11 @@ public class QInstanceValidator
assertCondition(StringUtils.hasContent(sqsQueueProvider.getSecretKey()), "Missing secretKey for SQSQueueProvider: " + name);
assertCondition(StringUtils.hasContent(sqsQueueProvider.getBaseURL()), "Missing baseURL for SQSQueueProvider: " + name);
assertCondition(StringUtils.hasContent(sqsQueueProvider.getRegion()), "Missing region for SQSQueueProvider: " + name);
if(assertCondition(sqsQueueProvider.getSchedule() != null, "Missing schedule for SQSQueueProvider: " + name))
{
validateScheduleMetaData(sqsQueueProvider.getSchedule(), qInstance, "SQSQueueProvider " + name + ", schedule: ");
}
}
});
}
@ -392,6 +401,11 @@ public class QInstanceValidator
{
assertCondition(Objects.equals(name, automationProvider.getName()), "Inconsistent naming for automationProvider: " + name + "/" + automationProvider.getName() + ".");
assertCondition(automationProvider.getType() != null, "Missing type for automationProvider: " + name);
if(assertCondition(automationProvider.getSchedule() != null, "Missing schedule for automationProvider: " + name))
{
validateScheduleMetaData(automationProvider.getSchedule(), qInstance, "automationProvider " + name + ", schedule: ");
}
});
}
}
@ -1316,7 +1330,7 @@ public class QInstanceValidator
if(process.getSchedule() != null)
{
QScheduleMetaData schedule = process.getSchedule();
assertCondition(schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null, "Either repeat millis or repeat seconds must be set on schedule in process " + processName);
validateScheduleMetaData(schedule, qInstance, "Process " + processName + ", schedule: ");
if(schedule.getVariantBackend() != null)
{
@ -1336,6 +1350,50 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void validateScheduleMetaData(QScheduleMetaData schedule, QInstance qInstance, String prefix)
{
boolean isRepeat = schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null;
boolean isCron = StringUtils.hasContent(schedule.getCronExpression());
assertCondition(isRepeat || isCron, prefix + " either repeatMillis or repeatSeconds or cronExpression must be set");
assertCondition(!(isRepeat && isCron), prefix + " both a repeat time and cronExpression may not be set");
if(isCron)
{
boolean hasDelay = schedule.getInitialDelayMillis() != null || schedule.getInitialDelaySeconds() != null;
assertCondition(!hasDelay, prefix + " a cron schedule may not have an initial delay");
try
{
CronExpression.validateExpression(schedule.getCronExpression());
}
catch(ParseException pe)
{
errors.add(prefix + " invalid cron expression: " + pe.getMessage());
}
if(assertCondition(StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a cron schedule must specify a cronTimeZoneId"))
{
String[] availableIDs = TimeZone.getAvailableIDs();
Optional<String> first = Arrays.stream(availableIDs).filter(id -> id.equals(schedule.getCronTimeZoneId())).findFirst();
assertCondition(first.isPresent(), prefix + " unrecognized cronTimeZoneId: " + schedule.getCronTimeZoneId());
}
}
else
{
assertCondition(!StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a non-cron schedule must not specify a cronTimeZoneId");
}
if(assertCondition(StringUtils.hasContent(schedule.getSchedulerName()), prefix + " is missing a scheduler name"))
{
assertCondition(qInstance.getScheduler(schedule.getSchedulerName()) != null, prefix + " is referencing an unknown scheduler name: " + schedule.getSchedulerName());
}
}
/*******************************************************************************
**
*******************************************************************************/

54
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java
View File

@ -53,6 +53,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@ -90,6 +91,7 @@ public class QInstance
private Map<String, QWidgetMetaDataInterface> widgets = new LinkedHashMap<>();
private Map<String, QQueueProviderMetaData> queueProviders = new LinkedHashMap<>();
private Map<String, QQueueMetaData> queues = new LinkedHashMap<>();
private Map<String, QSchedulerMetaData> schedulers = new LinkedHashMap<>();
private Map<String, QSupplementalInstanceMetaData> supplementalMetaData = new LinkedHashMap<>();
@ -1224,4 +1226,56 @@ public class QInstance
metaData.addSelfToInstance(this);
}
/*******************************************************************************
**
*******************************************************************************/
public void addScheduler(QSchedulerMetaData scheduler)
{
String name = scheduler.getName();
if(!StringUtils.hasContent(name))
{
throw (new IllegalArgumentException("Attempted to add a scheduler without a name."));
}
if(this.schedulers.containsKey(name))
{
throw (new IllegalArgumentException("Attempted to add a second scheduler with name: " + name));
}
this.schedulers.put(name, scheduler);
}
/*******************************************************************************
**
*******************************************************************************/
public QSchedulerMetaData getScheduler(String name)
{
return (this.schedulers.get(name));
}
/*******************************************************************************
** Getter for schedulers
**
*******************************************************************************/
public Map<String, QSchedulerMetaData> getSchedulers()
{
return schedulers;
}
/*******************************************************************************
** Setter for schedulers
**
*******************************************************************************/
public void setSchedulers(Map<String, QSchedulerMetaData> schedulers)
{
this.schedulers = schedulers;
}
}

109
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java
View File

@ -22,6 +22,9 @@
package com.kingsrook.qqq.backend.core.model.metadata.scheduleing;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** Meta-data to define scheduled actions within QQQ.
**
@ -37,16 +40,29 @@ public class QScheduleMetaData
{PARALLEL, SERIAL}
private String schedulerName;
private Integer repeatSeconds;
private Integer repeatMillis;
private Integer initialDelaySeconds;
private Integer initialDelayMillis;
private String cronExpression;
private String cronTimeZoneId;
private RunStrategy variantRunStrategy;
private String variantBackend;
/*******************************************************************************
**
*******************************************************************************/
public boolean isCron()
{
return StringUtils.hasContent(cronExpression);
}
/*******************************************************************************
** Getter for repeatSeconds
@ -244,4 +260,97 @@ public class QScheduleMetaData
return (this);
}
/*******************************************************************************
** Getter for cronExpression
*******************************************************************************/
public String getCronExpression()
{
return (this.cronExpression);
}
/*******************************************************************************
** Setter for cronExpression
*******************************************************************************/
public void setCronExpression(String cronExpression)
{
this.cronExpression = cronExpression;
}
/*******************************************************************************
** Fluent setter for cronExpression
*******************************************************************************/
public QScheduleMetaData withCronExpression(String cronExpression)
{
this.cronExpression = cronExpression;
return (this);
}
/*******************************************************************************
** Getter for cronTimeZoneId
*******************************************************************************/
public String getCronTimeZoneId()
{
return (this.cronTimeZoneId);
}
/*******************************************************************************
** Setter for cronTimeZoneId
*******************************************************************************/
public void setCronTimeZoneId(String cronTimeZoneId)
{
this.cronTimeZoneId = cronTimeZoneId;
}
/*******************************************************************************
** Fluent setter for cronTimeZoneId
*******************************************************************************/
public QScheduleMetaData withCronTimeZoneId(String cronTimeZoneId)
{
this.cronTimeZoneId = cronTimeZoneId;
return (this);
}
/*******************************************************************************
** Getter for schedulerName
*******************************************************************************/
public String getSchedulerName()
{
return (this.schedulerName);
}
/*******************************************************************************
** Setter for schedulerName
*******************************************************************************/
public void setSchedulerName(String schedulerName)
{
this.schedulerName = schedulerName;
}
/*******************************************************************************
** Fluent setter for schedulerName
*******************************************************************************/
public QScheduleMetaData withSchedulerName(String schedulerName)
{
this.schedulerName = schedulerName;
return (this);
}
}

131
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java Normal file
View File

@ -0,0 +1,131 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.scheduleing;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface;
/*******************************************************************************
**
*******************************************************************************/
public abstract class QSchedulerMetaData implements TopLevelMetaDataInterface
{
private String name;
private String type;
/*******************************************************************************
**
*******************************************************************************/
public boolean supportsCronSchedules()
{
return (false);
}
/*******************************************************************************
**
*******************************************************************************/
public abstract QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier<QSession> systemSessionSupplier) throws QException;
/*******************************************************************************
** Getter for name
*******************************************************************************/
public String getName()
{
return (this.name);
}
/*******************************************************************************
** Setter for name
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Fluent setter for name
*******************************************************************************/
public QSchedulerMetaData withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Getter for type
*******************************************************************************/
public String getType()
{
return (this.type);
}
/*******************************************************************************
** Setter for type
*******************************************************************************/
public void setType(String type)
{
this.type = type;
}
/*******************************************************************************
** Fluent setter for type
*******************************************************************************/
public QSchedulerMetaData withType(String type)
{
this.type = type;
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void addSelfToInstance(QInstance qInstance)
{
qInstance.addScheduler(this);
}
}

120
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java Normal file
View File

@ -0,0 +1,120 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz;
import java.util.Properties;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface;
import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler;
/*******************************************************************************
**
*******************************************************************************/
public class QuartzSchedulerMetaData extends QSchedulerMetaData
{
private static final QLogger LOG = QLogger.getLogger(QuartzSchedulerMetaData.class);
public static final String TYPE = "quartz";
private Properties properties;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public QuartzSchedulerMetaData()
{
setType(TYPE);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public boolean supportsCronSchedules()
{
return (true);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier<QSession> systemSessionSupplier) throws QException
{
try
{
QuartzScheduler quartzScheduler = QuartzScheduler.initInstance(qInstance, getName(), getProperties(), systemSessionSupplier);
return (quartzScheduler);
}
catch(Exception e)
{
LOG.error("Error initializing quartz scheduler", e);
throw (new QException("Error initializing quartz scheduler", e));
}
}
/*******************************************************************************
** Getter for properties
*******************************************************************************/
public Properties getProperties()
{
return (this.properties);
}
/*******************************************************************************
** Setter for properties
*******************************************************************************/
public void setProperties(Properties properties)
{
this.properties = properties;
}
/*******************************************************************************
** Fluent setter for properties
*******************************************************************************/
public QuartzSchedulerMetaData withProperties(Properties properties)
{
this.properties = properties;
return (this);
}
}

76
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java Normal file
View File

@ -0,0 +1,76 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface;
import com.kingsrook.qqq.backend.core.scheduler.simple.SimpleScheduler;
/*******************************************************************************
**
*******************************************************************************/
public class SimpleSchedulerMetaData extends QSchedulerMetaData
{
public static final String TYPE = "simple";
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public SimpleSchedulerMetaData()
{
setType(TYPE);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public boolean supportsCronSchedules()
{
return (false);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier<QSession> systemSessionSupplier)
{
SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance);
simpleScheduler.setSessionSupplier(systemSessionSupplier);
simpleScheduler.setSchedulerName(getName());
return simpleScheduler;
}
}

278
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java Normal file
View File

@ -0,0 +1,278 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.scheduler;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.context.CapturedContext;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
import com.kingsrook.qqq.backend.core.logging.LogPair;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
** QQQ service to manage scheduled jobs, using 1 or more Schedulers - implementations
** of the QSchedulerInterface
*******************************************************************************/
public class QScheduleManager
{
private static final QLogger LOG = QLogger.getLogger(QScheduleManager.class);
private static QScheduleManager qScheduleManager = null;
private final QInstance qInstance;
private final Supplier<QSession> systemUserSessionSupplier;
private Map<String, QSchedulerInterface> schedulers = new HashMap<>();
/*******************************************************************************
** Singleton constructor
*******************************************************************************/
private QScheduleManager(QInstance qInstance, Supplier<QSession> systemUserSessionSupplier)
{
this.qInstance = qInstance;
this.systemUserSessionSupplier = systemUserSessionSupplier;
}
/*******************************************************************************
** Singleton initiator - e.g., must be called to initially initialize the singleton
** before anyone else calls getInstance (they'll get an error if they call that first).
*******************************************************************************/
public static QScheduleManager initInstance(QInstance qInstance, Supplier<QSession> systemUserSessionSupplier)
{
if(qScheduleManager == null)
{
qScheduleManager = new QScheduleManager(qInstance, systemUserSessionSupplier);
}
return (qScheduleManager);
}
/*******************************************************************************
** Singleton accessor
*******************************************************************************/
public static QScheduleManager getInstance()
{
if(qScheduleManager == null)
{
throw (new IllegalStateException("QScheduleManager singleton has not been init'ed (call initInstance)."));
}
return (qScheduleManager);
}
/*******************************************************************************
**
*******************************************************************************/
public void start() throws QException
{
if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true))
{
LOG.info("Not starting ScheduleManager per settings.");
return;
}
/////////////////////////////////////////////////////////
// initialize the scheduler(s) we're configured to use //
/////////////////////////////////////////////////////////
for(QSchedulerMetaData schedulerMetaData : qInstance.getSchedulers().values())
{
QSchedulerInterface scheduler = schedulerMetaData.initSchedulerInstance(qInstance, systemUserSessionSupplier);
schedulers.put(schedulerMetaData.getName(), scheduler);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// ensure that everything which should be scheduled is scheduled, in the appropriate scheduler //
/////////////////////////////////////////////////////////////////////////////////////////////////
QContext.withTemporaryContext(new CapturedContext(qInstance, systemUserSessionSupplier.get()), () -> setupSchedules());
}
/*******************************************************************************
**
*******************************************************************************/
private void setupSchedules()
{
/////////////////////////////////////////////////////////
// let the schedulers know we're starting this process //
/////////////////////////////////////////////////////////
schedulers.values().forEach(s -> s.startOfSetupSchedules());
//////////////////////////////////
// schedule all queue providers //
//////////////////////////////////
for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values())
{
setupQueueProvider(queueProvider);
}
///////////////////////////////////////
// schedule all automation providers //
///////////////////////////////////////
for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values())
{
setupAutomationProviderPerTable(automationProvider);
}
/////////////////////////////////////////
// schedule all processes that need it //
/////////////////////////////////////////
for(QProcessMetaData process : qInstance.getProcesses().values())
{
if(process.getSchedule() != null)
{
QScheduleMetaData scheduleMetaData = process.getSchedule();
if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy()))
{
///////////////////////////////////////////////
// if no variants, or variant is serial mode //
///////////////////////////////////////////////
setupProcess(process, null);
}
else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy()))
{
/////////////////////////////////////////////////////////////////////////////////////////////////////
// if this a "parallel", which for example means we want to have a thread for each backend variant //
// running at the same time, get the variant records and schedule each separately //
/////////////////////////////////////////////////////////////////////////////////////////////////////
QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend());
for(QRecord qRecord : CollectionUtils.nonNullList(SchedulerUtils.getBackendVariantFilteredRecords(process)))
{
try
{
setupProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())));
}
catch(Exception e)
{
LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord));
}
}
}
else
{
LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided.");
}
}
}
/////////////////////////////////////////////////////////////
// todo - read dynamic schedules and schedule those things //
// e.g., user-scheduled processes, reports //
/////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////
// let the schedulers know we're done with this process //
//////////////////////////////////////////////////////////
schedulers.values().forEach(s -> s.endOfSetupSchedules());
}
/*******************************************************************************
**
*******************************************************************************/
private void setupProcess(QProcessMetaData process, Map<String, Serializable> backendVariantData)
{
QSchedulerInterface scheduler = getScheduler(process.getSchedule().getSchedulerName());
scheduler.setupProcess(process, backendVariantData, SchedulerUtils.allowedToStart(process));
}
/*******************************************************************************
**
*******************************************************************************/
private void setupQueueProvider(QQueueProviderMetaData queueProvider)
{
switch(queueProvider.getType())
{
case SQS:
setupSqsProvider((SQSQueueProviderMetaData) queueProvider);
break;
default:
throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType());
}
}
/*******************************************************************************
**
*******************************************************************************/
private void setupSqsProvider(SQSQueueProviderMetaData queueProvider)
{
QSchedulerInterface scheduler = getScheduler(queueProvider.getSchedule().getSchedulerName());
scheduler.setupSqsProvider(queueProvider, SchedulerUtils.allowedToStart(queueProvider));
}
/*******************************************************************************
**
*******************************************************************************/
private void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider)
{
QSchedulerInterface scheduler = getScheduler(automationProvider.getSchedule().getSchedulerName());
scheduler.setupAutomationProviderPerTable(automationProvider, SchedulerUtils.allowedToStart(automationProvider));
}
/*******************************************************************************
**
*******************************************************************************/
private QSchedulerInterface getScheduler(String schedulerName)
{
QSchedulerInterface scheduler = schedulers.get(schedulerName);
if(scheduler == null)
{
throw new NotImplementedException("default scheduler...");
}
return (scheduler);
}
}

84
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java Normal file
View File

@ -0,0 +1,84 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.scheduler;
import java.io.Serializable;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
/*******************************************************************************
**
*******************************************************************************/
public interface QSchedulerInterface
{
/*******************************************************************************
**
*******************************************************************************/
void setupSqsProvider(SQSQueueProviderMetaData queueProvider, boolean allowedToStart);
/*******************************************************************************
**
*******************************************************************************/
void stopAsync();
/*******************************************************************************
**
*******************************************************************************/
void stop();
/*******************************************************************************
**
*******************************************************************************/
void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStart);
/*******************************************************************************
**
*******************************************************************************/
void setupProcess(QProcessMetaData process, Map<String, Serializable> backendVariantData, boolean allowedToStart);
/*******************************************************************************
**
*******************************************************************************/
void start();
/*******************************************************************************
** let the scheduler know when the schedule manager is at the start of setting up schedules.
*******************************************************************************/
default void startOfSetupSchedules()
{
}
/*******************************************************************************
** let the scheduler know when the schedule manager is at the end of setting up schedules.
*******************************************************************************/
default void endOfSetupSchedules()
{
}
}

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java
View File

@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
@ -48,7 +49,7 @@ import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
/*******************************************************************************
**
** Utility methods used by various schedulers.
*******************************************************************************/
public class SchedulerUtils
{
@ -56,6 +57,16 @@ public class SchedulerUtils
/*******************************************************************************
**
*******************************************************************************/
public static boolean allowedToStart(TopLevelMetaDataInterface metaDataObject)
{
return (allowedToStart(metaDataObject.getName()));
}
/*******************************************************************************
**
*******************************************************************************/

32
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java
View File

@ -27,6 +27,7 @@ import java.util.Map;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils;
import org.quartz.DisallowConcurrentExecution;
import org.quartz.Job;
@ -57,16 +58,31 @@ public class QuartzRunProcessJob implements Job
JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap();
String processName = jobDataMap.getString("processName");
///////////////////////////////////
// todo - variants from job data //
///////////////////////////////////
Map<String, Serializable> backendVariantData = null;
LOG.debug("Running quartz process", logPair("processName", processName));
///////////////////////////////////////
// get the process from the instance //
///////////////////////////////////////
QInstance qInstance = QuartzScheduler.getInstance().getQInstance();
SchedulerUtils.runProcess(qInstance, QuartzScheduler.getInstance().getSessionSupplier(), qInstance.getProcess(processName), backendVariantData);
QProcessMetaData process = qInstance.getProcess(processName);
if(process == null)
{
LOG.warn("Could not find scheduled process in QInstance", logPair("processName", processName));
return;
}
///////////////////////////////////////////////
// if the job has variant data, get it ready //
///////////////////////////////////////////////
Map<String, Serializable> backendVariantData = null;
if(jobExecutionContext.getMergedJobDataMap().containsKey("backendVariantData"))
{
backendVariantData = (Map<String, Serializable>) jobExecutionContext.getMergedJobDataMap().get("backendVariantData");
}
/////////////
// run it. //
/////////////
LOG.debug("Running quartz process", logPair("processName", processName));
SchedulerUtils.runProcess(qInstance, QuartzScheduler.getInstance().getSessionSupplier(), qInstance.getProcess(processName), backendVariantData);
}
finally
{

399
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java
View File

@ -23,20 +23,38 @@ package com.kingsrook.qqq.backend.core.scheduler.quartz;
import java.io.Serializable;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.TimeZone;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface;
import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.AnyKey;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
import org.quartz.CronExpression;
import org.quartz.CronScheduleBuilder;
import org.quartz.Job;
import org.quartz.JobBuilder;
import org.quartz.JobDetail;
import org.quartz.JobKey;
import org.quartz.ScheduleBuilder;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.SimpleScheduleBuilder;
@ -49,41 +67,60 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
**
** Singleton to provide access between QQQ and the quartz Scheduler system.
*******************************************************************************/
public class QuartzScheduler
public class QuartzScheduler implements QSchedulerInterface
{
private static final QLogger LOG = QLogger.getLogger(QuartzScheduler.class);
private static QuartzScheduler quartzScheduler = null;
private final QInstance qInstance;
private String schedulerName;
private Properties quartzProperties;
private Supplier<QSession> sessionSupplier;
private Scheduler scheduler;
private Memoization<AnyKey, List<String>> jobGroupNamesMemoization = new Memoization<AnyKey, List<String>>()
.withTimeout(Duration.of(5, ChronoUnit.SECONDS));
private Memoization<String, Set<JobKey>> jobKeyNamesMemoization = new Memoization<String, Set<JobKey>>()
.withTimeout(Duration.of(5, ChronoUnit.SECONDS));
/*******************************************************************************
** Constructor
**
*******************************************************************************/
private QuartzScheduler(QInstance qInstance, Supplier<QSession> sessionSupplier)
private QuartzScheduler(QInstance qInstance, String schedulerName, Properties quartzProperties, Supplier<QSession> sessionSupplier)
{
this.qInstance = qInstance;
this.schedulerName = schedulerName;
this.quartzProperties = quartzProperties;
this.sessionSupplier = sessionSupplier;
}
/*******************************************************************************
** Singleton initiator...
** Singleton initiator - e.g., must be called to initially initialize the singleton
** before anyone else calls getInstance (they'll get an error if they call that first).
*******************************************************************************/
public static QuartzScheduler initInstance(QInstance qInstance, Supplier<QSession> sessionSupplier)
public static QuartzScheduler initInstance(QInstance qInstance, String schedulerName, Properties quartzProperties, Supplier<QSession> sessionSupplier) throws SchedulerException
{
if(quartzScheduler == null)
{
quartzScheduler = new QuartzScheduler(qInstance, sessionSupplier);
quartzScheduler = new QuartzScheduler(qInstance, schedulerName, quartzProperties, sessionSupplier);
///////////////////////////////////////////////////////////
// Grab the Scheduler instance from the Factory //
// initialize it with the properties we took in as input //
///////////////////////////////////////////////////////////
StdSchedulerFactory schedulerFactory = new StdSchedulerFactory();
schedulerFactory.initialize(quartzProperties);
quartzScheduler.scheduler = schedulerFactory.getScheduler();
}
return (quartzScheduler);
}
@ -97,7 +134,7 @@ public class QuartzScheduler
{
if(quartzScheduler == null)
{
throw (new IllegalStateException("QuartzScheduler singleton has not been init'ed."));
throw (new IllegalStateException("QuartzScheduler singleton has not been init'ed (call initInstance)."));
}
return (quartzScheduler);
}
@ -111,30 +148,6 @@ public class QuartzScheduler
{
try
{
// Properties properties = new Properties();
// properties.put("");
//////////////////////////////////////////////////
// Grab the Scheduler instance from the Factory //
//////////////////////////////////////////////////
StdSchedulerFactory schedulerFactory = new StdSchedulerFactory();
// schedulerFactory.initialize(properties);
this.scheduler = schedulerFactory.getScheduler();
////////////////////////////////////////
// todo - do we get our own property? //
////////////////////////////////////////
if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true))
{
LOG.info("Not starting QuartzScheduler per settings.");
return;
}
/////////////////////////////////////////////
// make sure all of our jobs are scheduled //
/////////////////////////////////////////////
scheduleAllJobs();
//////////////////////
// and start it off //
//////////////////////
@ -151,20 +164,166 @@ public class QuartzScheduler
/*******************************************************************************
**
*******************************************************************************/
private void scheduleAllJobs()
@Override
public void stop()
{
for(QProcessMetaData process : qInstance.getProcesses().values())
try
{
if(process.getSchedule() != null && SchedulerUtils.allowedToStart(process.getName()))
scheduler.shutdown(true);
}
catch(SchedulerException e)
{
if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy()))
LOG.error("Error shutting down (stopping) quartz scheduler", e);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void stopAsync()
{
scheduleProcess(process, null);
try
{
scheduler.shutdown(false);
}
catch(SchedulerException e)
{
LOG.error("Error shutting down (stopping) quartz scheduler", e);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void setupProcess(QProcessMetaData process, Map<String, Serializable> backendVariantData, boolean allowedToStart)
{
/////////////////////////
// set up job data map //
/////////////////////////
Map<String, Object> jobData = new HashMap<>();
jobData.put("processName", process.getName());
if(backendVariantData != null)
{
jobData.put("backendVariantData", backendVariantData);
}
scheduleJob(process.getName(), "processes", QuartzRunProcessJob.class, jobData, process.getSchedule(), allowedToStart);
}
/*******************************************************************************
**
*******************************************************************************/
private boolean scheduleJob(String jobName, String groupName, Class<? extends Job> jobClass, Map<String, Object> jobData, QScheduleMetaData scheduleMetaData, boolean allowedToStart)
{
try
{
/////////////////////////
// Define job instance //
/////////////////////////
JobKey jobKey = new JobKey(jobName, groupName);
JobDetail jobDetail = JobBuilder.newJob(jobClass)
.withIdentity(jobKey)
.storeDurably()
.requestRecovery()
.build();
jobDetail.getJobDataMap().putAll(jobData);
/////////////////////////////////////////////////////////
// map the qqq schedule meta data to a quartz schedule //
/////////////////////////////////////////////////////////
ScheduleBuilder<?> scheduleBuilder;
if(scheduleMetaData.isCron())
{
CronExpression cronExpression = new CronExpression(scheduleMetaData.getCronExpression());
cronExpression.setTimeZone(TimeZone.getTimeZone(scheduleMetaData.getCronTimeZoneId()));
scheduleBuilder = CronScheduleBuilder.cronSchedule(cronExpression);
}
else
{
LOG.error("Not yet know how to schedule parallel variant jobs");
long intervalMillis = Objects.requireNonNullElse(scheduleMetaData.getRepeatMillis(), scheduleMetaData.getRepeatSeconds() * 1000);
scheduleBuilder = SimpleScheduleBuilder.simpleSchedule()
.withIntervalInMilliseconds(intervalMillis)
.repeatForever();
}
Date startAt = new Date();
if(scheduleMetaData.getInitialDelayMillis() != null)
{
startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelayMillis());
}
else if(scheduleMetaData.getInitialDelaySeconds() != null)
{
startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelaySeconds() * 1000);
}
///////////////////////////////////////
// Define a Trigger for the schedule //
///////////////////////////////////////
Trigger trigger = TriggerBuilder.newTrigger()
.withIdentity(new TriggerKey(jobName, groupName))
.forJob(jobKey)
.withSchedule(scheduleBuilder)
.startAt(startAt)
.build();
///////////////////////////////////////
// Schedule the job with the trigger //
///////////////////////////////////////
addOrReplaceJobAndTrigger(jobKey, jobDetail, trigger);
//////////////////////////////////////////////////////////
// either pause or resume, based on if allowed to start //
//////////////////////////////////////////////////////////
if(!allowedToStart)
{
pauseJob(jobKey.getName(), jobKey.getGroup());
}
else
{
resumeJob(jobKey.getName(), jobKey.getGroup());
}
return (true);
}
catch(Exception e)
{
LOG.warn("Error scheduling job", e, logPair("name", jobName), logPair("group", groupName));
return (false);
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void setupSqsProvider(SQSQueueProviderMetaData sqsQueueProvider, boolean allowedToStartProvider)
{
for(QQueueMetaData queue : qInstance.getQueues().values())
{
boolean allowedToStart = allowedToStartProvider && SchedulerUtils.allowedToStart(queue.getName());
if(sqsQueueProvider.getName().equals(queue.getProviderName()))
{
/////////////////////////
// set up job data map //
/////////////////////////
Map<String, Object> jobData = new HashMap<>();
jobData.put("queueProviderName", sqsQueueProvider.getName());
jobData.put("queueName", queue.getName());
scheduleJob(queue.getName(), "sqsQueue", QuartzSqsPollerJob.class, jobData, sqsQueueProvider.getSchedule(), allowedToStart);
}
}
}
@ -174,132 +333,110 @@ public class QuartzScheduler
/*******************************************************************************
**
*******************************************************************************/
private void scheduleProcess(QProcessMetaData process, Map<String, Serializable> backendVariantData)
@Override
public void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStartProvider)
{
try
///////////////////////////////////////////////////////////////////////////////////
// ask the PollingAutomationPerTableRunner how many threads of itself need setup //
// then start a scheduled executor foreach one //
///////////////////////////////////////////////////////////////////////////////////
List<PollingAutomationPerTableRunner.TableActionsInterface> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName());
for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions)
{
QScheduleMetaData scheduleMetaData = process.getSchedule();
long intervalMillis = Objects.requireNonNullElse(scheduleMetaData.getRepeatMillis(), scheduleMetaData.getRepeatSeconds() * 1000);
Date startAt = new Date();
if(scheduleMetaData.getInitialDelayMillis() != null)
{
startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelayMillis());
}
if(scheduleMetaData.getInitialDelaySeconds() != null)
{
startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelaySeconds() * 1000);
}
boolean allowedToStart = allowedToStartProvider && SchedulerUtils.allowedToStart(tableAction.tableName());
/////////////////////////
// Define job instance //
// set up job data map //
/////////////////////////
JobKey jobKey = new JobKey(process.getName(), "processes");
JobDetail jobDetail = JobBuilder.newJob(QuartzRunProcessJob.class)
.withIdentity(jobKey)
.storeDurably()
.requestRecovery()
.build();
Map<String, Object> jobData = new HashMap<>();
jobData.put("automationProviderName", automationProvider.getName());
jobData.put("tableName", tableAction.tableName());
jobData.put("automationStatus", tableAction.status().toString());
jobDetail.getJobDataMap().put("processName", process.getName());
///////////////////////////////////////
// Define a Trigger for the schedule //
///////////////////////////////////////
Trigger trigger = TriggerBuilder.newTrigger()
.withIdentity(new TriggerKey(process.getName(), "processes"))
.forJob(jobKey)
.withSchedule(SimpleScheduleBuilder.simpleSchedule()
.withIntervalInMilliseconds(intervalMillis)
.repeatForever())
.startAt(startAt)
.build();
///////////////////////////////////////
// Schedule the job with the trigger //
///////////////////////////////////////
boolean isJobAlreadyScheduled = isJobAlreadyScheduled(jobKey);
if(isJobAlreadyScheduled)
{
this.scheduler.addJob(jobDetail, true);
this.scheduler.rescheduleJob(trigger.getKey(), trigger);
LOG.info("Re-scheduled process: " + process.getName());
}
else
{
this.scheduler.scheduleJob(jobDetail, trigger);
LOG.info("Scheduled new process: " + process.getName());
}
}
catch(Exception e)
{
LOG.warn("Error scheduling process", e, logPair("processName", process.getName()));
scheduleJob(tableAction.tableName() + "." + tableAction.status(), "tableAutomations", QuartzTableAutomationsJob.class, jobData, automationProvider.getSchedule(), allowedToStart);
}
}
/*******************************************************************************
** todo - probably rewrite this to not re-query quartz each time
**
*******************************************************************************/
private void addOrReplaceJobAndTrigger(JobKey jobKey, JobDetail jobDetail, Trigger trigger) throws SchedulerException
{
boolean isJobAlreadyScheduled = isJobAlreadyScheduled(jobKey);
if(isJobAlreadyScheduled)
{
this.scheduler.addJob(jobDetail, true);
this.scheduler.rescheduleJob(trigger.getKey(), trigger);
LOG.info("Re-scheduled job: " + jobKey);
}
else
{
this.scheduler.scheduleJob(jobDetail, trigger);
LOG.info("Scheduled new job: " + jobKey);
}
}
/*******************************************************************************
**
*******************************************************************************/
private boolean isJobAlreadyScheduled(JobKey jobKey) throws SchedulerException
{
for(String group : scheduler.getJobGroupNames())
Optional<List<String>> jobGroupNames = jobGroupNamesMemoization.getResult(AnyKey.getInstance(), (x) -> scheduler.getJobGroupNames());
if(jobGroupNames.isEmpty())
{
for(JobKey testJobKey : scheduler.getJobKeys(GroupMatcher.groupEquals(group)))
throw (new SchedulerException("Error getting job group names"));
}
for(String group : jobGroupNames.get())
{
if(testJobKey.equals(jobKey))
Optional<Set<JobKey>> jobKeys = jobKeyNamesMemoization.getResult(group, (x) -> scheduler.getJobKeys(GroupMatcher.groupEquals(group)));
if(jobKeys.isEmpty())
{
throw (new SchedulerException("Error getting job keys"));
}
if(jobKeys.get().contains(jobKey))
{
return (true);
}
}
}
return (false);
}
/*
private void todo() throws SchedulerException
/*******************************************************************************
**
*******************************************************************************/
private boolean deleteJob(JobKey jobKey)
{
// https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/ListJobs.html
// Listing all Jobs in the scheduler
for(String group : scheduler.getJobGroupNames())
try
{
for(JobKey jobKey : scheduler.getJobKeys(GroupMatcher.groupEquals(group)))
/////////////////////////////////////////////////////////////////////////////////////////////
// https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UnscheduleJob.html //
// Deleting a Job and Unscheduling All of Its Triggers //
/////////////////////////////////////////////////////////////////////////////////////////////
if(isJobAlreadyScheduled(jobKey))
{
System.out.println("Found job identified by: " + jobKey);
}
return scheduler.deleteJob(jobKey);
}
// https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UpdateJob.html
// Update an existing job
// Add the new job to the scheduler, instructing it to "replace"
// the existing job with the given name and group (if any)
JobDetail jobDetail = JobBuilder.newJob(QuartzRunProcessJob.class)
.withIdentity("job1", "group1")
.build();
// store, and set overwrite flag to 'true'
scheduler.addJob(jobDetail, true);
// https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UpdateTrigger.html
// Define a new Trigger
Trigger trigger = TriggerBuilder.newTrigger()
.withIdentity("newTrigger", "group1")
.startNow()
.build();
// tell the scheduler to remove the old trigger with the given key, and put the new one in its place
scheduler.rescheduleJob(new TriggerKey("oldTrigger", "group1"), trigger);
// https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UnscheduleJob.html
// Deleting a Job and Unscheduling All of Its Triggers
scheduler.deleteJob(new JobKey("job1", "group1"));
/////////////////////////////////////////
// return true to indicate, we're good //
/////////////////////////////////////////
return (true);
}
catch(Exception e)
{
LOG.warn("Error deleting job", e, logPair("jobKey", jobKey));
return false;
}
}
*/

79
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java Normal file
View File

@ -0,0 +1,79 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.scheduler.quartz;
import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import org.quartz.DisallowConcurrentExecution;
import org.quartz.Job;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
**
*******************************************************************************/
@DisallowConcurrentExecution
public class QuartzSqsPollerJob implements Job
{
private static final QLogger LOG = QLogger.getLogger(QuartzSqsPollerJob.class);
/*******************************************************************************
**
*******************************************************************************/
@Override
public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException
{
try
{
JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap();
String queueProviderName = jobDataMap.getString("queueProviderName");
String queueName = jobDataMap.getString("queueName");
QInstance qInstance = QuartzScheduler.getInstance().getQInstance();
SQSQueuePoller sqsQueuePoller = new SQSQueuePoller();
sqsQueuePoller.setQueueProviderMetaData((SQSQueueProviderMetaData) qInstance.getQueueProvider(queueProviderName));
sqsQueuePoller.setQueueMetaData(qInstance.getQueue(queueName));
sqsQueuePoller.setQInstance(qInstance);
sqsQueuePoller.setSessionSupplier(QuartzScheduler.getInstance().getSessionSupplier());
/////////////
// run it. //
/////////////
LOG.debug("Running quartz SQS Poller", logPair("queueName", queueName));
sqsQueuePoller.run();
}
finally
{
QContext.clear();
}
}
}

77
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java Normal file
View File

@ -0,0 +1,77 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.scheduler.quartz;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import org.quartz.DisallowConcurrentExecution;
import org.quartz.Job;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
**
*******************************************************************************/
@DisallowConcurrentExecution
public class QuartzTableAutomationsJob implements Job
{
private static final QLogger LOG = QLogger.getLogger(QuartzTableAutomationsJob.class);
/*******************************************************************************
**
*******************************************************************************/
@Override
public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException
{
try
{
JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap();
String tableName = jobDataMap.getString("tableName");
String automationProviderName = jobDataMap.getString("automationProviderName");
AutomationStatus automationStatus = AutomationStatus.valueOf(jobDataMap.getString("automationStatus"));
QInstance qInstance = QuartzScheduler.getInstance().getQInstance();
PollingAutomationPerTableRunner.TableActionsInterface tableAction = new PollingAutomationPerTableRunner.TableActions(tableName, automationStatus);
PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProviderName, QuartzScheduler.getInstance().getSessionSupplier(), tableAction);
/////////////
// run it. //
/////////////
LOG.debug("Running Table Automations", logPair("tableName", tableName), logPair("automationStatus", automationStatus));
runner.run();
}
finally
{
QContext.clear();
}
}
}

71
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDetailsPostQueryCustomizer.java Normal file
View File

@ -0,0 +1,71 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.scheduler.quartz.tables;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import org.apache.commons.lang3.SerializationUtils;
/*******************************************************************************
**
*******************************************************************************/
public class QuartzJobDetailsPostQueryCustomizer extends AbstractPostQueryCustomizer
{
private static final QLogger LOG = QLogger.getLogger(QuartzJobDetailsPostQueryCustomizer.class);
/*******************************************************************************
**
*******************************************************************************/
@Override
public List<QRecord> apply(List<QRecord> records)
{
for(QRecord record : records)
{
if(record.getValue("jobData") != null)
{
try
{
////////////////////////////////////////////////////////////////////////////////////////////////////////
// this field has a blob of essentially a serialized map - so, deserialize that, then convert to JSON //
////////////////////////////////////////////////////////////////////////////////////////////////////////
byte[] value = record.getValueByteArray("jobData");
Object deserialize = SerializationUtils.deserialize(value);
String json = JsonUtils.toJson(deserialize);
record.setValue("jobData", json);
}
catch(Exception e)
{
LOG.info("Error deserializing quartz job data", e);
}
}
}
return (records);
}
}

230
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java
View File

@ -1,6 +1,6 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
@ -19,7 +19,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.scheduler;
package com.kingsrook.qqq.backend.core.scheduler.simple;
import java.io.Serializable;
@ -30,22 +30,16 @@ import java.util.Objects;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner;
import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
import com.kingsrook.qqq.backend.core.logging.LogPair;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface;
import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils;
/*******************************************************************************
@ -58,12 +52,13 @@ import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
**
** All of these jobs run using a "system session" - as defined by the sessionSupplier.
*******************************************************************************/
public class ScheduleManager
public class SimpleScheduler implements QSchedulerInterface
{
private static final QLogger LOG = QLogger.getLogger(ScheduleManager.class);
private static final QLogger LOG = QLogger.getLogger(SimpleScheduler.class);
private static ScheduleManager scheduleManager = null;
private static SimpleScheduler simpleScheduler = null;
private final QInstance qInstance;
private String schedulerName;
protected Supplier<QSession> sessionSupplier;
@ -79,7 +74,7 @@ public class ScheduleManager
/*******************************************************************************
** Singleton constructor
*******************************************************************************/
private ScheduleManager(QInstance qInstance)
private SimpleScheduler(QInstance qInstance)
{
this.qInstance = qInstance;
}
@ -89,13 +84,13 @@ public class ScheduleManager
/*******************************************************************************
** Singleton accessor
*******************************************************************************/
public static ScheduleManager getInstance(QInstance qInstance)
public static SimpleScheduler getInstance(QInstance qInstance)
{
if(scheduleManager == null)
if(simpleScheduler == null)
{
scheduleManager = new ScheduleManager(qInstance);
simpleScheduler = new SimpleScheduler(qInstance);
}
return (scheduleManager);
return (simpleScheduler);
}
@ -103,88 +98,56 @@ public class ScheduleManager
/*******************************************************************************
**
*******************************************************************************/
@Override
public void start()
{
if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true))
for(StandardScheduledExecutor executor : executors)
{
executor.start();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void stopAsync()
{
for(StandardScheduledExecutor scheduledExecutor : executors)
{
scheduledExecutor.stopAsync();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void stop()
{
for(StandardScheduledExecutor scheduledExecutor : executors)
{
scheduledExecutor.stop();
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStartProvider)
{
if(!allowedToStartProvider)
{
LOG.info("Not starting ScheduleManager per settings.");
return;
}
boolean needToClearContext = false;
try
{
if(QContext.getQInstance() == null)
{
needToClearContext = true;
QContext.init(qInstance, sessionSupplier.get());
}
for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values())
{
startQueueProvider(queueProvider);
}
for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values())
{
startAutomationProviderPerTable(automationProvider);
}
for(QProcessMetaData process : qInstance.getProcesses().values())
{
if(process.getSchedule() != null && SchedulerUtils.allowedToStart(process.getName()))
{
QScheduleMetaData scheduleMetaData = process.getSchedule();
if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy()))
{
///////////////////////////////////////////////
// if no variants, or variant is serial mode //
///////////////////////////////////////////////
startProcess(process, null);
}
else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy()))
{
/////////////////////////////////////////////////////////////////////////////////////////////////////
// if this a "parallel", which for example means we want to have a thread for each backend variant //
// running at the same time, get the variant records and schedule each separately //
/////////////////////////////////////////////////////////////////////////////////////////////////////
QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend());
for(QRecord qRecord : CollectionUtils.nonNullList(SchedulerUtils.getBackendVariantFilteredRecords(process)))
{
try
{
startProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())));
}
catch(Exception e)
{
LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord));
}
}
}
else
{
LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided.");
}
}
}
}
finally
{
if(needToClearContext)
{
QContext.clear();
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private void startAutomationProviderPerTable(QAutomationProviderMetaData automationProvider)
{
///////////////////////////////////////////////////////////////////////////////////
// ask the PollingAutomationPerTableRunner how many threads of itself need setup //
// then start a scheduled executor foreach one //
@ -201,11 +164,6 @@ public class ScheduleManager
executor.setName(runner.getName());
setScheduleInExecutor(schedule, executor);
if(!executor.start())
{
LOG.warn("executor.start return false for: " + executor.getName());
}
executors.add(executor);
}
}
@ -216,28 +174,14 @@ public class ScheduleManager
/*******************************************************************************
**
*******************************************************************************/
private void startQueueProvider(QQueueProviderMetaData queueProvider)
@Override
public void setupSqsProvider(SQSQueueProviderMetaData queueProvider, boolean allowedToStartProvider)
{
if(SchedulerUtils.allowedToStart(queueProvider.getName()))
if(!allowedToStartProvider)
{
switch(queueProvider.getType())
{
case SQS:
startSqsProvider((SQSQueueProviderMetaData) queueProvider);
break;
default:
throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType());
}
}
return;
}
/*******************************************************************************
**
*******************************************************************************/
private void startSqsProvider(SQSQueueProviderMetaData queueProvider)
{
QInstance scheduleManagerQueueInstance = qInstance;
Supplier<QSession> scheduleManagerSessionSupplier = sessionSupplier;
@ -259,11 +203,6 @@ public class ScheduleManager
executor.setName(queue.getName());
setScheduleInExecutor(schedule, executor);
if(!executor.start())
{
LOG.warn("executor.start return false for: " + executor.getName());
}
executors.add(executor);
}
}
@ -274,8 +213,14 @@ public class ScheduleManager
/*******************************************************************************
**
*******************************************************************************/
private void startProcess(QProcessMetaData process, Map<String, Serializable> backendVariantData)
@Override
public void setupProcess(QProcessMetaData process, Map<String, Serializable> backendVariantData, boolean allowedToStart)
{
if(!allowedToStart)
{
return;
}
Runnable runProcess = () ->
{
SchedulerUtils.runProcess(qInstance, sessionSupplier, process, backendVariantData);
@ -284,11 +229,6 @@ public class ScheduleManager
StandardScheduledExecutor executor = new StandardScheduledExecutor(runProcess);
executor.setName("process:" + process.getName());
setScheduleInExecutor(process.getSchedule(), executor);
if(!executor.start())
{
LOG.warn("executor.start return false for: " + executor.getName());
}
executors.add(executor);
}
@ -363,22 +303,40 @@ public class ScheduleManager
/*******************************************************************************
**
*******************************************************************************/
public void stopAsync()
static void resetSingleton()
{
for(StandardScheduledExecutor scheduledExecutor : executors)
{
scheduledExecutor.stopAsync();
}
simpleScheduler = null;
}
/*******************************************************************************
**
** Getter for schedulerName
*******************************************************************************/
static void resetSingleton()
public String getSchedulerName()
{
scheduleManager = null;
return (this.schedulerName);
}
/*******************************************************************************
** Setter for schedulerName
*******************************************************************************/
public void setSchedulerName(String schedulerName)
{
this.schedulerName = schedulerName;
}
/*******************************************************************************
** Fluent setter for schedulerName
*******************************************************************************/
public SimpleScheduler withSchedulerName(String schedulerName)
{
this.schedulerName = schedulerName;
return (this);
}
}

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java
View File

@ -1,6 +1,6 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
@ -19,7 +19,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.scheduler;
package com.kingsrook.qqq.backend.core.scheduler.simple;
import java.util.concurrent.Executors;

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java
View File

@ -43,7 +43,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.scheduler.StandardScheduledExecutor;
import com.kingsrook.qqq.backend.core.scheduler.simple.StandardScheduledExecutor;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;

123
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java
View File

@ -28,6 +28,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
@ -69,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMeta
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
@ -367,6 +369,127 @@ class QInstanceValidatorTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void test_validateSchedules()
{
String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE;
Supplier<QScheduleMetaData> baseScheduleMetaData = () -> new QScheduleMetaData()
.withSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME);
////////////////////////////////////////////////////
// do our basic schedule validations on a process //
////////////////////////////////////////////////////
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()),
"either repeatMillis or repeatSeconds or cronExpression must be set");
String validCronString = "* * * * * ?";
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
.withRepeatMillis(1)
.withCronExpression(validCronString)
.withCronTimeZoneId("UTC")),
"both a repeat time and cronExpression may not be set");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
.withRepeatSeconds(1)
.withCronExpression(validCronString)
.withCronTimeZoneId("UTC")),
"both a repeat time and cronExpression may not be set");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
.withRepeatSeconds(1)
.withRepeatMillis(1)
.withCronExpression(validCronString)
.withCronTimeZoneId("UTC")),
"both a repeat time and cronExpression may not be set");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
.withInitialDelaySeconds(1)
.withCronExpression(validCronString)
.withCronTimeZoneId("UTC")),
"cron schedule may not have an initial delay");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
.withInitialDelayMillis(1)
.withCronExpression(validCronString)
.withCronTimeZoneId("UTC")),
"cron schedule may not have an initial delay");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
.withInitialDelaySeconds(1)
.withInitialDelayMillis(1)
.withCronExpression(validCronString)
.withCronTimeZoneId("UTC")),
"cron schedule may not have an initial delay");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
.withCronExpression(validCronString)),
"must specify a cronTimeZoneId");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
.withCronExpression(validCronString)
.withCronTimeZoneId("foobar")),
"unrecognized cronTimeZoneId: foobar");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
.withCronExpression("* * * * * *")
.withCronTimeZoneId("UTC")),
"invalid cron expression: Support for specifying both");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
.withCronExpression("x")
.withCronTimeZoneId("UTC")),
"invalid cron expression: Illegal cron expression format");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
.withRepeatSeconds(10)
.withCronTimeZoneId("UTC")),
"non-cron schedule must not specify a cronTimeZoneId");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
.withSchedulerName(null)
.withCronExpression(validCronString)
.withCronTimeZoneId("UTC")),
"is missing a scheduler name");
assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()
.withSchedulerName("not-a-scheduler")
.withCronExpression(validCronString)
.withCronTimeZoneId("UTC")),
"referencing an unknown scheduler name: not-a-scheduler");
/////////////////////////////////
// validate some success cases //
/////////////////////////////////
assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withRepeatSeconds(1)));
assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withRepeatMillis(1)));
assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("UTC")));
assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("America/New_York")));
//////////////////////////////////////////////////////////////////
// make sure automation providers get their schedules validated //
//////////////////////////////////////////////////////////////////
assertValidationFailureReasons((qInstance) -> qInstance.getAutomationProvider(TestUtils.POLLING_AUTOMATION).withSchedule(baseScheduleMetaData.get()
.withSchedulerName(null)
.withCronExpression(validCronString)
.withCronTimeZoneId("UTC")),
"is missing a scheduler name");
/////////////////////////////////////////////////////////////
// make sure queue providers get their schedules validated //
/////////////////////////////////////////////////////////////
assertValidationFailureReasons((qInstance) -> ((SQSQueueProviderMetaData)qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER)).withSchedule(baseScheduleMetaData.get()
.withSchedulerName(null)
.withCronExpression(validCronString)
.withCronTimeZoneId("UTC")),
"is missing a scheduler name");
}
/*******************************************************************************
** Test that a table with no fields fails.
**

63
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java Normal file
View File

@ -0,0 +1,63 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.scheduler.quartz;
import java.util.Properties;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.quartz.SchedulerException;
/*******************************************************************************
** Unit test for QuartzScheduler
*******************************************************************************/
class QuartzSchedulerTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
void beforeEach() throws SchedulerException
{
Properties quartzProperties = new Properties();
quartzProperties.put("", "");
quartzProperties.put("org.quartz.scheduler.instanceName", "TestScheduler");
quartzProperties.put("org.quartz.threadPool.threadCount", "3");
quartzProperties.put("org.quartz.jobStore.class", "org.quartz.simpl.RAMJobStore");
QuartzScheduler.initInstance(QContext.getQInstance(), "TestScheduler", quartzProperties, QContext::getQSession);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test()
{
}
}

27
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java → qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java
View File

@ -1,6 +1,6 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
@ -19,7 +19,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.scheduler;
package com.kingsrook.qqq.backend.core.scheduler.simple;
import java.util.List;
@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
@ -48,7 +49,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for ScheduleManager
*******************************************************************************/
class ScheduleManagerTest extends BaseTest
class SimpleSchedulerTest extends BaseTest
{
/*******************************************************************************
@ -57,7 +58,7 @@ class ScheduleManagerTest extends BaseTest
@AfterEach
void afterEach()
{
ScheduleManager.resetSingleton();
SimpleScheduler.resetSingleton();
}
@ -69,12 +70,13 @@ class ScheduleManagerTest extends BaseTest
void testStartAndStop()
{
QInstance qInstance = QContext.getQInstance();
ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance);
scheduleManager.start();
SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance);
simpleScheduler.setSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME);
simpleScheduler.start();
assertThat(scheduleManager.getExecutors()).isNotEmpty();
assertThat(simpleScheduler.getExecutors()).isNotEmpty();
scheduleManager.stopAsync();
simpleScheduler.stopAsync();
}
@ -101,11 +103,12 @@ class ScheduleManagerTest extends BaseTest
BasicStep.counter = 0;
ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance);
scheduleManager.setSessionSupplier(QSession::new);
scheduleManager.start();
SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance);
simpleScheduler.setSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME);
simpleScheduler.setSessionSupplier(QSession::new);
simpleScheduler.start();
SleepUtils.sleep(50, TimeUnit.MILLISECONDS);
scheduleManager.stopAsync();
simpleScheduler.stopAsync();
System.out.println("Ran: " + BasicStep.counter + " times");
assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice");

27
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -92,6 +92,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple.SimpleSchedulerMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
@ -176,6 +179,8 @@ public class TestUtils
public static final String SECURITY_KEY_TYPE_STORE_NULL_BEHAVIOR = "storeNullBehavior";
public static final String SECURITY_KEY_TYPE_INTERNAL_OR_EXTERNAL = "internalOrExternal";
public static final String SIMPLE_SCHEDULER_NAME = "simpleScheduler";
/*******************************************************************************
@ -237,11 +242,23 @@ public class TestUtils
defineWidgets(qInstance);
defineApps(qInstance);
qInstance.addScheduler(defineSimpleScheduler());
return (qInstance);
}
/*******************************************************************************
**
*******************************************************************************/
private static QSchedulerMetaData defineSimpleScheduler()
{
return new SimpleSchedulerMetaData().withName(SIMPLE_SCHEDULER_NAME);
}
/*******************************************************************************
**
*******************************************************************************/
@ -349,7 +366,10 @@ public class TestUtils
private static QAutomationProviderMetaData definePollingAutomationProvider()
{
return (new PollingAutomationProviderMetaData()
.withName(POLLING_AUTOMATION));
.withName(POLLING_AUTOMATION)
.withSchedule(new QScheduleMetaData()
.withSchedulerName(SIMPLE_SCHEDULER_NAME)
.withRepeatSeconds(60)));
}
@ -1313,7 +1333,10 @@ public class TestUtils
.withAccessKey(accessKey)
.withSecretKey(secretKey)
.withRegion(region)
.withBaseURL(baseURL));
.withBaseURL(baseURL)
.withSchedule(new QScheduleMetaData()
.withRepeatSeconds(60)
.withSchedulerName(SIMPLE_SCHEDULER_NAME)));
}