diff --git a/docs/index.adoc b/docs/index.adoc index 1ab41a32..d777f0cc 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -26,6 +26,16 @@ include::metaData/Reports.adoc[leveloffset=+1] include::metaData/Icons.adoc[leveloffset=+1] include::metaData/PermissionRules.adoc[leveloffset=+1] +== Services + +include::misc/ScheduledJobs.adoc[leveloffset=+1] + +=== Web server (Javalin) +#todo# + +=== API server (OpenAPI) +#todo# + == Custom Application Code include::misc/QContext.adoc[leveloffset=+1] include::misc/QRecords.adoc[leveloffset=+1] @@ -63,4 +73,4 @@ include::implementations/TableSync.adoc[leveloffset=+1] // later... include::actions/RenderTemplateAction.adoc[leveloffset=+1] == QQQ Utility Classes -include::utilities/RecordLookupHelper.adoc[leveloffset=+1] \ No newline at end of file +include::utilities/RecordLookupHelper.adoc[leveloffset=+1] diff --git a/docs/misc/ScheduledJobs.adoc b/docs/misc/ScheduledJobs.adoc new file mode 100644 index 00000000..b4a88a61 --- /dev/null +++ b/docs/misc/ScheduledJobs.adoc @@ -0,0 +1,141 @@ +== Schedulers and Scheduled Jobs +include::../variables.adoc[] + +QQQ has the ability to automatically run various types of jobs on schedules, +either defined in your instance's meta-data, +or optionally via data in your application, in a `scheduledJob` table. + +=== Schedulers and QSchedulerMetaData +2 types of schedulers are included in QQQ by default (though an application can define its own schedulers): + +* `SimpleScheduler` - is (as its name suggests) a simple class which uses java's `ScheduledExecutorService` +to run jobs on repeating intervals. +** Cannot run cron schedules - only repeating intervals. +** If multiple servers are running, each will potentially run the same job concurrently +** Has no configurations, e.g., to limit the number of threads. + +* `QuartzScheduler` - uses the 3rd party https://www.quartz-scheduler.org/[Quartz Scheduler] library to provide +a much more capable, though admittedly more complex, scheduling solution. +** Can run both cron schedules and repeating intervals. +** By default, will not allow concurrent executions of the same job. +** Supports multiple configurations, e.g., to limit the number of threads. + +An application can define its own scheduler by providing a class which implements the `QSchedulerInterface`. + +A `QInstance` can work with 0 or more schedulers, as defined by adding `QSchedulerMetaData` objects +to the instance. + +This meta-data class is `abstract`, and is extended by the 2 built-in schedulers +(e.g., `SimpleSchedulerMetaData` and `QuartzSchedulerMetaData`). As such, +these concrete subclasses are what you need to instantiate and add to your instance. + +To configure a QuartzScheduler, you can add a `Properties` object to the `QuartzSchedulerMetaData` object. +See https://www.quartz-scheduler.org/documentation/[Quartz's documentation] for available configuration properties. + +[source,java] +.Defining SchedulerMetaData +---- +qInstance.addScheduler(new SimpleSchedulerMetaData().withName("mySimpleScheduler")); + +qInstance.addScheduler(new QuartzSchedulerMetaData() + .withName("myQuartzScheduler") + .withProperties(myQuartzProperties); +---- + +=== SchedulableTypes +The types of jobs which can be scheduled in a QQQ application are defined in the `QInstance` by +instances of the `SchedulableType` meta-data class. +These objects contain a name, along with a `QCodeReference` to the `runner`, +which must be a class that implements the `SchedulableRunner` interface. + +By default, (in the `QInstanceEnricher`), QQQ will make 3 `SchedulableType` options available: + +* `PROCESS` - Any Process defined in the `QInstance` can be scheduled. +* `QUEUE_PROCESSOR` - A Queue defined in the `QInstance`, which requires polling (e.g., SQS), can be scheduled. +* `TABLE_AUTOMATIONS` - A Table in the `QInstance`, with `AutomationDetails` referring to an +AutomationProvider which requires polling, can be scheduled. + +If an application only wants to use a subset of these `SchedulableType` options, +or to add custom `SchedulableType` options, +the `QInstance` will need to have 1 or more `SchedulableType` objects in it before the `QInstanceEnricher` runs. + +=== User-defined Scheduled Jobs +To allow users to schedule jobs (rather than using programmer-defined schedules (in meta-data)), +you can add a set of tables to your `QInstance`, using the `ScheduledJobsMetaDataProvider` class: + +[source,java] +.Adding the ScheduledJob tables and related meta-data to a QInstance +---- +new ScheduledJobsMetaDataProvider().defineAll( + qInstance, backendName, table -> tableEnricher(table)); +---- + +This meta-data provider adds a "scheduledJob" and "scheduledJobParameter" table, along with +some PossibleValueSources. +These tables include post-action customizers, which manage (re-, un-) scheduling jobs based on +changes made to records in this these tables. + +Also, when `QScheduleManager` is started, it will query these tables,and will schedule jobs as defined therein. + +_You can use a mix of user-defined and meta-data defined scheduled jobs in your instance. +However, if a ScheduledJob record references a process, queue, or table automation with a +meta-data defined schedule, the ScheduledJob will NOT be started by ScheduleManager -- +rather, the meta-data definition will "win"._ + +[source,sql] +.Example of inserting scheduled jobs records directly into an SQL backend +---- +-- A process: +INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES + ('myProcess', 'QuartzScheduler', null, null, 300, 'PROCESS', 1); +INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES + ((SELECT id FROM scheduled_job WHERE label = 'myProcess'), 'processName', 'myProcess'); + +-- A table's insert & update automations: +INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES + ('myTable.PENDING_INSERT_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1), + ('myTable.PENDING_UPDATE_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1); +INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES + ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'tableName', 'myTable'), + ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'automationStatus', 'PENDING_INSERT_AUTOMATIONS'), + ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'tableName', 'myTable'), + ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'automationStatus', 'PENDING_UPDATE_AUTOMATIONS'); + +-- A queue processor: +INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES + ('mySqsQueue', 'QuartzScheduler', null, null, 60, 'QUEUE_PROCESSOR', 1); +INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES + ((SELECT id FROM scheduled_job WHERE label = 'mySqsQueue'), 'queueName', 'mySqsQueue'); +---- + +=== Running Scheduled Jobs +In a server running QQQ, if you wish to start running scheduled jobs, you need to initialize +the `QScheduleManger` singleton class, then call its `start()` method. + +Note that internally, this class will check for a system property of `qqq.scheduleManager.enabled` +or an environment variable of `QQQ_SCHEDULE_MANAGER_ENABLED`, and if the first value found is +`"false"`, then the scheduler will not actually run its jobs (but, in the case of the `QuartzSchdeuler`, +it will be available for managing scheduled jobs). + +The method `QScheduleManager.initInstance` requires 2 parameters: Your `QInstance`, and a +`Supplier` lambda, which returns the session that will be used for scheduled jobs when they +are executed. + +[source,java] +.Starting the Schedule Manager service +---- +QScheduleManager.initInstance(qInstance, () -> systemUserSession).start(); +---- + +=== Examples +[source,java] +.Attach a schedule in meta-data to a Process +---- +QProcessMetaData myProcess = new QProcessMetaData() + // ... + .withSchedule(new QScheduleMetaData() + .withSchedulerName("myScheduler") + .withDescription("Run myProcess every five minutes") + .withRepeatSeconds(300)) +---- + diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index 58191fa6..38bad7f5 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -163,6 +163,19 @@ 1.12.321 + + org.quartz-scheduler + quartz + 2.3.2 + + + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.23.0 + + org.apache.maven.plugins diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java index e4871048..6cc317d5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java @@ -159,7 +159,7 @@ public class AsyncJobManager private T runAsyncJob(String jobName, AsyncJob asyncJob, UUIDAndTypeStateKey uuidAndTypeStateKey, AsyncJobStatus asyncJobStatus) { String originalThreadName = Thread.currentThread().getName(); - Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8)); + Thread.currentThread().setName("Job:" + jobName); try { LOG.debug("Starting job " + uuidAndTypeStateKey.getUuid()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java index 2113facc..9b013a85 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java @@ -132,6 +132,11 @@ public class PollingAutomationPerTableRunner implements Runnable *******************************************************************************/ String tableName(); + /******************************************************************************* + ** + *******************************************************************************/ + QTableAutomationDetails tableAutomationDetails(); + /******************************************************************************* ** *******************************************************************************/ @@ -143,7 +148,7 @@ public class PollingAutomationPerTableRunner implements Runnable /******************************************************************************* ** Wrapper for a pair of (tableName, automationStatus) *******************************************************************************/ - public record TableActions(String tableName, AutomationStatus status) implements TableActionsInterface + public record TableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status) implements TableActionsInterface { /******************************************************************************* ** @@ -159,7 +164,7 @@ public class PollingAutomationPerTableRunner implements Runnable ** extended version of TableAction, for sharding use-case - adds the shard ** details. *******************************************************************************/ - public record ShardedTableActions(String tableName, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface + public record ShardedTableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface { /******************************************************************************* ** @@ -198,8 +203,8 @@ public class PollingAutomationPerTableRunner implements Runnable { Serializable shardId = record.getValue(automationDetails.getShardIdFieldName()); String label = record.getValueString(automationDetails.getShardLabelFieldName()); - tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label)); - tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label)); + tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label)); + tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label)); } } catch(Exception e) @@ -209,11 +214,11 @@ public class PollingAutomationPerTableRunner implements Runnable } else { - /////////////////////////////////////////////////////////////////// - // for non-sharded, we just need tabler name & automation status // - /////////////////////////////////////////////////////////////////// - tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS)); - tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + ////////////////////////////////////////////////////////////////// + // for non-sharded, we just need table name & automation status // + ////////////////////////////////////////////////////////////////// + tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java index edf9a9cf..52864385 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java @@ -47,12 +47,24 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** records that the delete action marked in error - the user might want to do ** something special with them (idk, try some other way to delete them?) *******************************************************************************/ -public abstract class AbstractPostDeleteCustomizer +public abstract class AbstractPostDeleteCustomizer implements TableCustomizerInterface { protected DeleteInput deleteInput; + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postDelete(DeleteInput deleteInput, List records) throws QException + { + this.deleteInput = deleteInput; + return apply(records); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java index c7e2bfc6..100fe267 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java @@ -42,12 +42,24 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** ** Note that the full insertInput is available as a field in this class. *******************************************************************************/ -public abstract class AbstractPostInsertCustomizer +public abstract class AbstractPostInsertCustomizer implements TableCustomizerInterface { protected InsertInput insertInput; + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + this.insertInput = insertInput; + return (apply(records)); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java index d1beaa4c..669aa06b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java @@ -23,16 +23,29 @@ package com.kingsrook.qqq.backend.core.actions.customizers; import java.util.List; -import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; import com.kingsrook.qqq.backend.core.model.data.QRecord; /******************************************************************************* ** *******************************************************************************/ -public abstract class AbstractPostQueryCustomizer +public abstract class AbstractPostQueryCustomizer implements TableCustomizerInterface { - protected AbstractTableActionInput input; + protected QueryOrGetInputInterface input; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postQuery(QueryOrGetInputInterface queryInput, List records) throws QException + { + input = queryInput; + return apply(records); + } @@ -47,7 +60,7 @@ public abstract class AbstractPostQueryCustomizer ** Getter for input ** *******************************************************************************/ - public AbstractTableActionInput getInput() + public QueryOrGetInputInterface getInput() { return (input); } @@ -58,7 +71,7 @@ public abstract class AbstractPostQueryCustomizer ** Setter for input ** *******************************************************************************/ - public void setInput(AbstractTableActionInput input) + public void setInput(QueryOrGetInputInterface input) { this.input = input; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java index b0d55b35..53e00583 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -48,7 +49,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** available (if the backend supports it) - both as a list (`getOldRecordList`) ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`). *******************************************************************************/ -public abstract class AbstractPostUpdateCustomizer +public abstract class AbstractPostUpdateCustomizer implements TableCustomizerInterface { protected UpdateInput updateInput; protected List oldRecordList; @@ -57,6 +58,19 @@ public abstract class AbstractPostUpdateCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + this.updateInput = updateInput; + this.oldRecordList = oldRecordList.orElse(null); + return apply(records); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java index 4b848a14..80460a86 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java @@ -50,7 +50,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** Note that the full deleteInput is available as a field in this class. ** *******************************************************************************/ -public abstract class AbstractPreDeleteCustomizer +public abstract class AbstractPreDeleteCustomizer implements TableCustomizerInterface { protected DeleteInput deleteInput; @@ -58,6 +58,19 @@ public abstract class AbstractPreDeleteCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preDelete(DeleteInput deleteInput, List records, boolean isPreview) throws QException + { + this.deleteInput = deleteInput; + this.isPreview = isPreview; + return apply(records); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java index 196ea4b8..c0706a5c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java @@ -47,7 +47,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** ** Note that the full insertInput is available as a field in this class. *******************************************************************************/ -public abstract class AbstractPreInsertCustomizer +public abstract class AbstractPreInsertCustomizer implements TableCustomizerInterface { protected InsertInput insertInput; @@ -70,6 +70,30 @@ public abstract class AbstractPreInsertCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + this.insertInput = insertInput; + this.isPreview = isPreview; + return (apply(records)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview) + { + return getWhenToRun(); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java index b8a95ed2..701ce30c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -53,7 +54,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** available (if the backend supports it) - both as a list (`getOldRecordList`) ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`). *******************************************************************************/ -public abstract class AbstractPreUpdateCustomizer +public abstract class AbstractPreUpdateCustomizer implements TableCustomizerInterface { protected UpdateInput updateInput; protected List oldRecordList; @@ -63,6 +64,20 @@ public abstract class AbstractPreUpdateCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + this.updateInput = updateInput; + this.isPreview = isPreview; + this.oldRecordList = oldRecordList.orElse(null); + return apply(records); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java index 05acf79a..14773be9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.actions.customizers; +import java.lang.reflect.Constructor; import java.util.Optional; import java.util.function.Function; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; @@ -34,42 +35,35 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* ** Utility to load code for running QQQ customizers. + ** + ** TODO - redo all to go through method that memoizes class & constructor + ** lookup. That memoziation causes 1,000,000 such calls to go from ~500ms + ** to ~100ms. *******************************************************************************/ public class QCodeLoader { private static final QLogger LOG = QLogger.getLogger(QCodeLoader.class); - - - /******************************************************************************* - ** - *******************************************************************************/ - public static Optional> getTableCustomizerFunction(QTableMetaData table, String customizerName) - { - Optional codeReference = table.getCustomizer(customizerName); - if(codeReference.isPresent()) - { - return (Optional.ofNullable(QCodeLoader.getFunction(codeReference.get()))); - } - return (Optional.empty()); - } + private static Memoization> constructorMemoization = new Memoization<>(); /******************************************************************************* ** *******************************************************************************/ - public static Optional getTableCustomizer(Class expectedClass, QTableMetaData table, String customizerName) + public static Optional getTableCustomizer(QTableMetaData table, String customizerName) { Optional codeReference = table.getCustomizer(customizerName); if(codeReference.isPresent()) { - return (Optional.ofNullable(QCodeLoader.getAdHoc(expectedClass, codeReference.get()))); + return (Optional.ofNullable(QCodeLoader.getAdHoc(TableCustomizerInterface.class, codeReference.get()))); } return (Optional.empty()); } @@ -175,8 +169,21 @@ public class QCodeLoader try { - Class customizerClass = Class.forName(codeReference.getName()); - return ((T) customizerClass.getConstructor().newInstance()); + Optional> constructor = constructorMemoization.getResultThrowing(codeReference.getName(), (UnsafeFunction, Exception>) s -> + { + Class customizerClass = Class.forName(codeReference.getName()); + return customizerClass.getConstructor(); + }); + + if(constructor.isPresent()) + { + return ((T) constructor.get().newInstance()); + } + else + { + LOG.error("Could not get constructor for code reference", logPair("codeReference", codeReference)); + return (null); + } } catch(Exception e) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java new file mode 100644 index 00000000..3a7cfa41 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java @@ -0,0 +1,202 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.customizers; + + +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Common interface used by all (core) TableCustomizer types (e.g., post-query, + ** and {pre,post}-{insert,update,delete}. + ** + ** Note that the abstract-base classes for each action still exist, though have + ** been back-ported to be implementors of this interface. The action classes + ** will now expect this type, and call this type's methods. + ** + *******************************************************************************/ +public interface TableCustomizerInterface +{ + QLogger LOG = QLogger.getLogger(TableCustomizerInterface.class); + + + /******************************************************************************* + ** custom actions to run after a query (or get!) takes place. + ** + *******************************************************************************/ + default List postQuery(QueryOrGetInputInterface queryInput, List records) throws QException + { + LOG.info("A default implementation of postQuery is running... Probably not expected!", logPair("tableName", queryInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** custom actions before an insert takes place. + ** + ** It's important for implementations to be aware of the isPreview field, which + ** is set to true when the code is running to give users advice, e.g., on a review + ** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing + ** things like storing data, you don't want to do that if isPreview is true!! + ** + ** General implementation would be, to iterate over the records (the inputs to + ** the insert action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records + ** - possibly manipulating values (`setValue`) + ** - possibly throwing an exception - if you really don't want the insert operation to continue. + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class. + *******************************************************************************/ + default List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + LOG.info("A default implementation of preInsert is running... Probably not expected!", logPair("tableName", insertInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview) + { + return (AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS); + } + + + /******************************************************************************* + ** custom actions after an insert takes place. + ** + ** General implementation would be, to iterate over the records (the outputs of + ** the insert action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records + ** - possibly throwing an exception - though doing so won't stop the update, and instead + ** will just set a warning on all of the updated records... + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go back to the caller. + *******************************************************************************/ + default List postInsert(InsertInput insertInput, List records) throws QException + { + LOG.info("A default implementation of postInsert is running... Probably not expected!", logPair("tableName", insertInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** custom actions before an update takes place. + ** + ** It's important for implementations to be aware of the isPreview field, which + ** is set to true when the code is running to give users advice, e.g., on a review + ** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing + ** things like storing data, you don't want to do that if isPreview is true!! + ** + ** General implementation would be, to iterate over the records (the inputs to + ** the update action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records + ** - possibly manipulating values (`setValue`) + ** - possibly throwing an exception - if you really don't want the update operation to continue. + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class. + ** + ** Note, "old records" (e.g., with values freshly fetched from the backend) will be + ** available (if the backend supports it) + *******************************************************************************/ + default List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + LOG.info("A default implementation of preUpdate is running... Probably not expected!", logPair("tableName", updateInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** custom actions after an update takes place. + ** + ** General implementation would be, to iterate over the records (the outputs of + ** the update action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records? + ** - possibly throwing an exception - though doing so won't stop the update, and instead + ** will just set a warning on all of the updated records... + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go back to the caller. + ** + ** Note, "old records" (e.g., with values freshly fetched from the backend) will be + ** available (if the backend supports it). + *******************************************************************************/ + default List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + LOG.info("A default implementation of postUpdate is running... Probably not expected!", logPair("tableName", updateInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** Custom actions before a delete takes place. + ** + ** It's important for implementations to be aware of the isPreview param, which + ** is set to true when the code is running to give users advice, e.g., on a review + ** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing + ** things like storing data, you don't want to do that if isPreview is true!! + ** + ** General implementation would be, to iterate over the records (which the DeleteAction + ** would look up based on the inputs to the delete action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records + ** - possibly throwing an exception - if you really don't want the delete operation to continue. + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) - this is how errors + ** and warnings are propagated to the DeleteAction. Note that any records with + ** an error will NOT proceed to the backend's delete interface - but those with + ** warnings will. + *******************************************************************************/ + default List preDelete(DeleteInput deleteInput, List records, boolean isPreview) throws QException + { + LOG.info("A default implementation of preDelete is running... Probably not expected!", logPair("tableName", deleteInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** Custom actions after a delete takes place. + ** + ** General implementation would be, to iterate over the records (ones which didn't + ** have a delete error), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records? + ** - possibly throwing an exception - though doing so won't stop the delete, and instead + ** will just set a warning on all of the deleted records... + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go back + ** to the caller - this is how errors and warnings are propagated . + *******************************************************************************/ + default List postDelete(DeleteInput deleteInput, List records) throws QException + { + LOG.info("A default implementation of postDelete is running... Probably not expected!", logPair("tableName", deleteInput.getTableName())); + return (records); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java index 2c753b56..4c4d0f8d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java @@ -29,13 +29,13 @@ package com.kingsrook.qqq.backend.core.actions.customizers; *******************************************************************************/ public enum TableCustomizers { - POST_QUERY_RECORD("postQueryRecord", AbstractPostQueryCustomizer.class), - PRE_INSERT_RECORD("preInsertRecord", AbstractPreInsertCustomizer.class), - POST_INSERT_RECORD("postInsertRecord", AbstractPostInsertCustomizer.class), - PRE_UPDATE_RECORD("preUpdateRecord", AbstractPreUpdateCustomizer.class), - POST_UPDATE_RECORD("postUpdateRecord", AbstractPostUpdateCustomizer.class), - PRE_DELETE_RECORD("preDeleteRecord", AbstractPreDeleteCustomizer.class), - POST_DELETE_RECORD("postDeleteRecord", AbstractPostDeleteCustomizer.class); + POST_QUERY_RECORD("postQueryRecord", TableCustomizerInterface.class), + PRE_INSERT_RECORD("preInsertRecord", TableCustomizerInterface.class), + POST_INSERT_RECORD("postInsertRecord", TableCustomizerInterface.class), + PRE_UPDATE_RECORD("preUpdateRecord", TableCustomizerInterface.class), + POST_UPDATE_RECORD("postUpdateRecord", TableCustomizerInterface.class), + PRE_DELETE_RECORD("preDeleteRecord", TableCustomizerInterface.class), + POST_DELETE_RECORD("postDeleteRecord", TableCustomizerInterface.class); private final String role; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java new file mode 100644 index 00000000..5a9f67ce --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java @@ -0,0 +1,202 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** Generic widget that does an aggregate query, and presents its results + ** as a table, using group-by values as both row & column labels. + *******************************************************************************/ +public class Aggregate2DTableWidgetRenderer extends AbstractWidgetRenderer +{ + private static final QLogger LOG = QLogger.getLogger(Aggregate2DTableWidgetRenderer.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + Map values = input.getWidgetMetaData().getDefaultValues(); + + String tableName = ValueUtils.getValueAsString(values.get("tableName")); + String valueField = ValueUtils.getValueAsString(values.get("valueField")); + String rowField = ValueUtils.getValueAsString(values.get("rowField")); + String columnField = ValueUtils.getValueAsString(values.get("columnField")); + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + AggregateInput aggregateInput = new AggregateInput(); + aggregateInput.setTableName(tableName); + + // todo - allow input of "list of columns" (e.g., in case some miss sometimes, or as a version of filter) + // todo - max rows, max cols? + + // todo - from input map + QQueryFilter filter = new QQueryFilter(); + aggregateInput.setFilter(filter); + + Aggregate aggregate = new Aggregate(valueField, AggregateOperator.COUNT); + aggregateInput.withAggregate(aggregate); + + GroupBy rowGroupBy = new GroupBy(table.getField(rowField)); + GroupBy columnGroupBy = new GroupBy(table.getField(columnField)); + aggregateInput.withGroupBy(rowGroupBy); + aggregateInput.withGroupBy(columnGroupBy); + + String orderBys = ValueUtils.getValueAsString(values.get("orderBys")); + if(StringUtils.hasContent(orderBys)) + { + for(String orderBy : orderBys.split(",")) + { + switch(orderBy) + { + case "row" -> filter.addOrderBy(new QFilterOrderByGroupBy(rowGroupBy)); + case "column" -> filter.addOrderBy(new QFilterOrderByGroupBy(columnGroupBy)); + case "value" -> filter.addOrderBy(new QFilterOrderByAggregate(aggregate)); + default -> LOG.warn("Unrecognized orderBy: " + orderBy); + } + } + } + + AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput); + + Map> data = new LinkedHashMap<>(); + Set columnsSet = new LinkedHashSet<>(); + + for(AggregateResult result : aggregateOutput.getResults()) + { + Serializable column = result.getGroupByValue(columnGroupBy); + Serializable row = result.getGroupByValue(rowGroupBy); + Serializable value = result.getAggregateValue(aggregate); + + Map rowMap = data.computeIfAbsent(row, (k) -> new LinkedHashMap<>()); + rowMap.put(column, value); + columnsSet.add(column); + } + + // todo - possible values from rows, cols + + //////////////////////////////////// + // setup datastructures for table // + //////////////////////////////////// + List> tableRows = new ArrayList<>(); + List tableColumns = new ArrayList<>(); + tableColumns.add(new TableData.Column("default", table.getField(rowField).getLabel(), "_row", "2fr", "left")); + + for(Serializable column : columnsSet) + { + tableColumns.add(new TableData.Column("default", String.valueOf(column) /* todo display value */, String.valueOf(column), "1fr", "right")); + } + + tableColumns.add(new TableData.Column("default", "Total", "_total", "1fr", "right")); + + TableData tableData = new TableData(null, tableColumns, tableRows) + .withRowsPerPage(100) + .withFixedStickyLastRow(false) + .withHidePaginationDropdown(true); + + Map columnSums = new HashMap<>(); + int grandTotal = 0; + for(Map.Entry> rowEntry : data.entrySet()) + { + Map rowMap = new HashMap<>(); + tableRows.add(rowMap); + + rowMap.put("_row", rowEntry.getKey() /* todo display value */); + int rowTotal = 0; + for(Serializable column : columnsSet) + { + Serializable value = rowEntry.getValue().get(column); + if(value == null) + { + value = 0; // todo? + } + + Integer valueAsInteger = Objects.requireNonNullElse(ValueUtils.getValueAsInteger(value), 0); + rowTotal += valueAsInteger; + columnSums.putIfAbsent(column, 0); + columnSums.put(column, columnSums.get(column) + valueAsInteger); + + rowMap.put(String.valueOf(column), value); // todo format commas? + } + + rowMap.put("_total", rowTotal); + grandTotal += rowTotal; + } + + /////////////// + // total row // + /////////////// + Map totalRowMap = new HashMap<>(); + tableRows.add(totalRowMap); + + totalRowMap.put("_row", "Total"); + int rowTotal = 0; + for(Serializable column : columnsSet) + { + Serializable value = columnSums.get(column); + if(value == null) + { + value = 0; // todo? + } + + totalRowMap.put(String.valueOf(column), value); // todo format commas? + } + + totalRowMap.put("_total", grandTotal); + + return (new RenderWidgetOutput(tableData)); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java index b6f69d4c..e5b2e956 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java @@ -137,7 +137,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer *******************************************************************************/ public Builder withCanAddChildRecord(boolean b) { - widgetMetaData.withDefaultValue("canAddChildRecord", true); + widgetMetaData.withDefaultValue("canAddChildRecord", b); return (this); } @@ -151,6 +151,17 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer widgetMetaData.withDefaultValue("disabledFieldsForNewChildRecords", new HashSet<>(disabledFieldsForNewChildRecords)); return (this); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withManageAssociationName(String manageAssociationName) + { + widgetMetaData.withDefaultValue("manageAssociationName", manageAssociationName); + return (this); + } } @@ -178,52 +189,60 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows")); } - //////////////////////////////////////////////////////// - // fetch the record that we're getting children for. // - // e.g., the left-side of the join, with the input id // - //////////////////////////////////////////////////////// - GetInput getInput = new GetInput(); - getInput.setTableName(join.getLeftTable()); - getInput.setPrimaryKey(id); - GetOutput getOutput = new GetAction().execute(getInput); - QRecord record = getOutput.getRecord(); - - if(record == null) + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // fetch the record that we're getting children for. e.g., the left-side of the join, with the input id // + // but - only try this if we were given an id. note, this widget could be called for on an INSERT screen, where we don't have a record yet // + // but we still want to be able to return all the other data in here that otherwise comes from the widget meta data, join, etc. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + int totalRows = 0; + QRecord primaryRecord = null; + QQueryFilter filter = null; + QueryOutput queryOutput = new QueryOutput(new QueryInput()); + if(StringUtils.hasContent(id)) { - throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id)); - } + GetInput getInput = new GetInput(); + getInput.setTableName(join.getLeftTable()); + getInput.setPrimaryKey(id); + GetOutput getOutput = new GetAction().execute(getInput); + primaryRecord = getOutput.getRecord(); - //////////////////////////////////////////////////////////////////// - // set up the query - for the table on the right side of the join // - //////////////////////////////////////////////////////////////////// - QQueryFilter filter = new QQueryFilter(); - for(JoinOn joinOn : join.getJoinOns()) - { - filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(record.getValue(joinOn.getLeftField())))); - } - filter.setOrderBys(join.getOrderBys()); - filter.setLimit(maxRows); + if(primaryRecord == null) + { + throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id)); + } - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(join.getRightTable()); - queryInput.setShouldTranslatePossibleValues(true); - queryInput.setShouldGenerateDisplayValues(true); - queryInput.setFilter(filter); - QueryOutput queryOutput = new QueryAction().execute(queryInput); + //////////////////////////////////////////////////////////////////// + // set up the query - for the table on the right side of the join // + //////////////////////////////////////////////////////////////////// + filter = new QQueryFilter(); + for(JoinOn joinOn : join.getJoinOns()) + { + filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(primaryRecord.getValue(joinOn.getLeftField())))); + } + filter.setOrderBys(join.getOrderBys()); + filter.setLimit(maxRows); - QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords()); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(join.getRightTable()); + queryInput.setShouldTranslatePossibleValues(true); + queryInput.setShouldGenerateDisplayValues(true); + queryInput.setFilter(filter); + queryOutput = new QueryAction().execute(queryInput); - int totalRows = queryOutput.getRecords().size(); - if(maxRows != null && (queryOutput.getRecords().size() == maxRows)) - { - ///////////////////////////////////////////////////////////////////////////////////// - // if the input said to only do some max, and the # of results we got is that max, // - // then do a count query, for displaying 1-n of // - ///////////////////////////////////////////////////////////////////////////////////// - CountInput countInput = new CountInput(); - countInput.setTableName(join.getRightTable()); - countInput.setFilter(filter); - totalRows = new CountAction().execute(countInput).getCount(); + QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords()); + + totalRows = queryOutput.getRecords().size(); + if(maxRows != null && (queryOutput.getRecords().size() == maxRows)) + { + ///////////////////////////////////////////////////////////////////////////////////// + // if the input said to only do some max, and the # of results we got is that max, // + // then do a count query, for displaying 1-n of // + ///////////////////////////////////////////////////////////////////////////////////// + CountInput countInput = new CountInput(); + countInput.setTableName(join.getRightTable()); + countInput.setFilter(filter); + totalRows = new CountAction().execute(countInput).getCount(); + } } String tablePath = input.getInstance().getTablePath(rightTable.getName()); @@ -239,10 +258,14 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer // new child records must have values from the join-ons // ////////////////////////////////////////////////////////// Map defaultValuesForNewChildRecords = new HashMap<>(); - for(JoinOn joinOn : join.getJoinOns()) + if(primaryRecord != null) { - defaultValuesForNewChildRecords.put(joinOn.getRightField(), record.getValue(joinOn.getLeftField())); + for(JoinOn joinOn : join.getJoinOns()) + { + defaultValuesForNewChildRecords.put(joinOn.getRightField(), primaryRecord.getValue(joinOn.getLeftField())); + } } + widgetData.setDefaultValuesForNewChildRecords(defaultValuesForNewChildRecords); Map widgetValues = input.getWidgetMetaData().getDefaultValues(); @@ -250,6 +273,22 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer { widgetData.setDisabledFieldsForNewChildRecords((Set) widgetValues.get("disabledFieldsForNewChildRecords")); } + else + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there are no disabled fields specified - then normally any fields w/ a default value get implicitly disabled // + // but - if we didn't look-up the primary record, then we'll want to explicit disable fields from joins // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(primaryRecord == null) + { + Set implicitlyDisabledFields = new HashSet<>(); + widgetData.setDisabledFieldsForNewChildRecords(implicitlyDisabledFields); + for(JoinOn joinOn : join.getJoinOns()) + { + implicitlyDisabledFields.add(joinOn.getRightField()); + } + } + } } return (new RenderWidgetOutput(widgetData)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java index 39090e2f..57a368f7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java @@ -34,15 +34,18 @@ import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.permissions.DenyBehavior; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -333,9 +336,25 @@ public class PermissionsHelper QPermissionRules rules = getEffectivePermissionRules(tableMetaData, instance); String baseName = getEffectivePermissionBaseName(rules, tableMetaData.getName()); - for(TablePermissionSubType permissionSubType : TablePermissionSubType.values()) + QBackendMetaData backend = instance.getBackend(tableMetaData.getBackendName()); + if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_INSERT)) { - addEffectiveAvailablePermission(rules, permissionSubType, rs, baseName, tableMetaData, "Table"); + addEffectiveAvailablePermission(rules, TablePermissionSubType.INSERT, rs, baseName, tableMetaData, "Table"); + } + + if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_UPDATE)) + { + addEffectiveAvailablePermission(rules, TablePermissionSubType.EDIT, rs, baseName, tableMetaData, "Table"); + } + + if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_DELETE)) + { + addEffectiveAvailablePermission(rules, TablePermissionSubType.DELETE, rs, baseName, tableMetaData, "Table"); + } + + if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_QUERY) || tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_GET)) + { + addEffectiveAvailablePermission(rules, TablePermissionSubType.READ, rs, baseName, tableMetaData, "Table"); } } @@ -369,7 +388,10 @@ public class PermissionsHelper { QPermissionRules rules = getEffectivePermissionRules(widgetMetaData, instance); String baseName = getEffectivePermissionBaseName(rules, widgetMetaData.getName()); - addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget"); + if(!rules.getLevel().equals(PermissionLevel.NOT_PROTECTED)) + { + addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget"); + } } return (rs); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index c9350e68..8774d9ae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -497,10 +497,10 @@ public class RunProcessAction ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if backend specifies that it uses variants, look for that data in the session and append to our basepull key // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(process.getSchedule() != null && process.getSchedule().getVariantBackend() != null) + if(process.getSchedule() != null && process.getVariantBackend() != null) { QSession session = QContext.getQSession(); - QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getSchedule().getVariantBackend()); + QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend()); if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue())) { LOG.info("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java index 93b9d183..4f1fc32b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java @@ -73,6 +73,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.aggregates.AggregatesInterface; import com.kingsrook.qqq.backend.core.utils.aggregates.BigDecimalAggregates; import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates; +import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates; /******************************************************************************* @@ -553,6 +554,12 @@ public class GenerateReportAction AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates()); fieldAggregates.add(record.getValueInteger(field.getName())); } + else if(field.getType().equals(QFieldType.LONG)) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new LongAggregates()); + fieldAggregates.add(record.getValueLong(field.getName())); + } else if(field.getType().equals(QFieldType.DECIMAL)) { @SuppressWarnings("unchecked") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java index ee49499a..05cc83b1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java @@ -35,9 +35,8 @@ import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostDeleteCustomizer; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreDeleteCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper; @@ -250,7 +249,7 @@ public class DeleteAction ////////////////////////////////////////////////////////////// // finally, run the post-delete customizer, if there is one // ////////////////////////////////////////////////////////////// - Optional postDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPostDeleteCustomizer.class, table, TableCustomizers.POST_DELETE_RECORD.getRole()); + Optional postDeleteCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_DELETE_RECORD.getRole()); if(postDeleteCustomizer.isPresent() && oldRecordList.isPresent()) { //////////////////////////////////////////////////////////////////////////// @@ -260,8 +259,7 @@ public class DeleteAction try { - postDeleteCustomizer.get().setDeleteInput(deleteInput); - List postCustomizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer); + List postCustomizerResult = postDeleteCustomizer.get().postDelete(deleteInput, recordsForCustomizer); /////////////////////////////////////////////////////// // check if any records got errors in the customizer // @@ -327,13 +325,11 @@ public class DeleteAction /////////////////////////////////////////////////////////////////////////// // after all validations, run the pre-delete customizer, if there is one // /////////////////////////////////////////////////////////////////////////// - Optional preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole()); + Optional preDeleteCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_DELETE_RECORD.getRole()); List customizerResult = oldRecordList.get(); if(preDeleteCustomizer.isPresent()) { - preDeleteCustomizer.get().setDeleteInput(deleteInput); - preDeleteCustomizer.get().setIsPreview(isPreview); - customizerResult = preDeleteCustomizer.get().apply(oldRecordList.get()); + customizerResult = preDeleteCustomizer.get().preDelete(deleteInput, oldRecordList.get(), isPreview); } ///////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java index 728f5874..bd110c32 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java @@ -27,8 +27,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.ActionHelper; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface; import com.kingsrook.qqq.backend.core.actions.tables.helpers.GetActionCacheHelper; @@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; /******************************************************************************* @@ -57,7 +58,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; *******************************************************************************/ public class GetAction { - private Optional postGetRecordCustomizer; + private Optional postGetRecordCustomizer; private GetInput getInput; private QPossibleValueTranslator qPossibleValueTranslator; @@ -87,7 +88,7 @@ public class GetAction throw (new QException("Requested to Get a record from an unrecognized table: " + getInput.getTableName())); } - postGetRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole()); + postGetRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole()); this.getInput = getInput; QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); @@ -107,9 +108,11 @@ public class GetAction } GetOutput getOutput; + boolean usingDefaultGetInterface = false; if(getInterface == null) { getInterface = new DefaultGetInterface(); + usingDefaultGetInterface = true; } getInterface.validateInput(getInput); @@ -123,10 +126,11 @@ public class GetAction new GetActionCacheHelper().handleCaching(getInput, getOutput); } - //////////////////////////////////////////////////////// - // if the record is found, perform post-actions on it // - //////////////////////////////////////////////////////// - if(getOutput.getRecord() != null) + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the record is found, perform post-actions on it // + // unless the defaultGetInterface was used - as it just does a query, and the query will do the post-actions. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(getOutput.getRecord() != null && !usingDefaultGetInterface) { getOutput.setRecord(postRecordActions(getOutput.getRecord())); } @@ -202,7 +206,7 @@ public class GetAction } else { - throw (new QException("No primaryKey or uniqueKey was passed to Get")); + throw (new QException("Unable to get " + ObjectUtils.tryElse(() -> queryInput.getTable().getLabel(), queryInput.getTableName()) + ". Missing required input.")); } queryInput.setFilter(filter); @@ -216,12 +220,12 @@ public class GetAction ** Run the necessary actions on a record. This may include setting display values, ** translating possible values, and running post-record customizations. *******************************************************************************/ - public QRecord postRecordActions(QRecord record) + public QRecord postRecordActions(QRecord record) throws QException { QRecord returnRecord = record; if(this.postGetRecordCustomizer.isPresent()) { - returnRecord = postGetRecordCustomizer.get().apply(List.of(record)).get(0); + returnRecord = postGetRecordCustomizer.get().postQuery(getInput, List.of(record)).get(0); } if(getInput.getShouldTranslatePossibleValues()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index ea3df9c0..9215c861 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -37,9 +38,9 @@ import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; @@ -168,13 +169,12 @@ public class InsertAction extends AbstractQActionFunction postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole()); + Optional postInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_INSERT_RECORD.getRole()); if(postInsertCustomizer.isPresent()) { try { - postInsertCustomizer.get().setInsertInput(insertInput); - insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords())); + insertOutput.setRecords(postInsertCustomizer.get().postInsert(insertInput, insertOutput.getRecords())); } catch(Exception e) { @@ -232,31 +232,29 @@ public class InsertAction extends AbstractQActionFunction preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole()); + Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); if(preInsertCustomizer.isPresent()) { - preInsertCustomizer.get().setInsertInput(insertInput); - preInsertCustomizer.get().setIsPreview(isPreview); - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS); } setDefaultValuesInRecords(table, insertInput.getRecords()); ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords(), null); - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS); setErrorsIfUniqueKeyErrors(insertInput, table); - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS); if(insertInput.getInputSource().shouldValidateRequiredFields()) { validateRequiredFields(insertInput); } - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS); ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT); - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS); } @@ -290,13 +288,13 @@ public class InsertAction extends AbstractQActionFunction preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException + private void runPreInsertCustomizerIfItIsTime(InsertInput insertInput, boolean isPreview, Optional preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException { if(preInsertCustomizer.isPresent()) { - if(whenToRun.equals(preInsertCustomizer.get().getWhenToRun())) + if(whenToRun.equals(preInsertCustomizer.get().whenToRunPreInsert(insertInput, isPreview))) { - insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords())); + insertInput.setRecords(preInsertCustomizer.get().preInsert(insertInput, insertInput.getRecords(), isPreview)); } } } @@ -321,7 +319,7 @@ public class InsertAction extends AbstractQActionFunction postQueryRecordCustomizer; + private Optional postQueryRecordCustomizer; private QueryInput queryInput; private QueryInterface queryInterface; @@ -100,7 +100,7 @@ public class QueryAction } QBackendMetaData backend = queryInput.getBackend(); - postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole()); + postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole()); this.queryInput = queryInput; if(queryInput.getRecordPipe() != null) @@ -264,7 +264,7 @@ public class QueryAction { if(this.postQueryRecordCustomizer.isPresent()) { - records = postQueryRecordCustomizer.get().apply(records); + records = postQueryRecordCustomizer.get().postQuery(queryInput, records); } if(queryInput.getShouldTranslatePossibleValues()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java index a69acc45..6a92cf80 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java @@ -34,9 +34,8 @@ import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostUpdateCustomizer; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreUpdateCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper; @@ -192,14 +191,12 @@ public class UpdateAction ////////////////////////////////////////////////////////////// // finally, run the post-update customizer, if there is one // ////////////////////////////////////////////////////////////// - Optional postUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPostUpdateCustomizer.class, table, TableCustomizers.POST_UPDATE_RECORD.getRole()); + Optional postUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_UPDATE_RECORD.getRole()); if(postUpdateCustomizer.isPresent()) { try { - postUpdateCustomizer.get().setUpdateInput(updateInput); - oldRecordList.ifPresent(l -> postUpdateCustomizer.get().setOldRecordList(l)); - updateOutput.setRecords(postUpdateCustomizer.get().apply(updateOutput.getRecords())); + updateOutput.setRecords(postUpdateCustomizer.get().postUpdate(updateInput, updateOutput.getRecords(), oldRecordList)); } catch(Exception e) { @@ -273,13 +270,10 @@ public class UpdateAction /////////////////////////////////////////////////////////////////////////// // after all validations, run the pre-update customizer, if there is one // /////////////////////////////////////////////////////////////////////////// - Optional preUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPreUpdateCustomizer.class, table, TableCustomizers.PRE_UPDATE_RECORD.getRole()); + Optional preUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_UPDATE_RECORD.getRole()); if(preUpdateCustomizer.isPresent()) { - preUpdateCustomizer.get().setUpdateInput(updateInput); - preUpdateCustomizer.get().setIsPreview(isPreview); - oldRecordList.ifPresent(l -> preUpdateCustomizer.get().setOldRecordList(l)); - updateInput.setRecords(preUpdateCustomizer.get().apply(updateInput.getRecords())); + updateInput.setRecords(preUpdateCustomizer.get().preUpdate(updateInput, updateInput.getRecords(), isPreview, oldRecordList)); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java index 12a3f2f9..a6a23d86 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java @@ -269,6 +269,10 @@ public class QPossibleValueTranslator { value = ValueUtils.getValueAsInteger(value); } + if(field.getType().equals(QFieldType.LONG) && !(value instanceof Long)) + { + value = ValueUtils.getValueAsLong(value); + } } catch(QValueException e) { @@ -366,6 +370,14 @@ public class QPossibleValueTranslator *******************************************************************************/ private String translatePossibleValueCustom(Serializable value, QPossibleValueSource possibleValueSource) { + ///////////////////////////////// + // null input gets null output // + ///////////////////////////////// + if(value == null) + { + return (null); + } + try { QCustomPossibleValueProvider customPossibleValueProvider = QCodeLoader.getCustomPossibleValueProvider(possibleValueSource); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index 5b176bbc..726ceea4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -490,6 +490,13 @@ public class QValueFormatter { adornmentValues = fileDownloadAdornment.get().getValues(); } + else + { + /////////////////////////////////////////////////////// + // don't change blobs unless they are file-downloads // + /////////////////////////////////////////////////////// + continue; + } String fileNameField = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD)); String fileNameFormat = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java index 1d1a52bb..7cf1aea6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java @@ -34,5 +34,11 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; *******************************************************************************/ public record CapturedContext(QInstance qInstance, QSession qSession, QBackendTransaction qBackendTransaction, Stack actionStack) { - + /******************************************************************************* + ** Simpler constructor + *******************************************************************************/ + public CapturedContext(QInstance qInstance, QSession qSession) + { + this(qInstance, qSession, null, null); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java index 69cb0dbd..cfc931b4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeVoidVoidMethod; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -54,6 +55,7 @@ public class QContext private static ThreadLocal> objectsThreadLocal = new ThreadLocal<>(); + /******************************************************************************* ** private constructor - class is not meant to be instantiated. *******************************************************************************/ @@ -105,6 +107,25 @@ public class QContext + /******************************************************************************* + ** + *******************************************************************************/ + public static void withTemporaryContext(CapturedContext context, UnsafeVoidVoidMethod method) throws T + { + CapturedContext originalContext = QContext.capture(); + try + { + QContext.init(context); + method.run(); + } + finally + { + QContext.init(originalContext); + } + } + + + /******************************************************************************* ** Init a new thread with the context captured from a different thread. e.g., ** when starting some async task. @@ -267,6 +288,7 @@ public class QContext } + /******************************************************************************* ** get one named object from the Context for the current thread. may return null. *******************************************************************************/ @@ -280,6 +302,7 @@ public class QContext } + /******************************************************************************* ** get one named object from the Context for the current thread, cast to the ** specified type if possible. if not found, or wrong type, empty is returned. diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 232ade6c..4bcac23f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -77,6 +77,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.Bulk import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -159,6 +160,18 @@ public class QInstanceEnricher } enrichJoins(); + + ////////////////////////////////////////////////////////////////////////////// + // if the instance DOES have 1 or more scheduler, but no schedulable types, // + // then go ahead and add the default set that qqq knows about // + ////////////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(qInstance.getSchedulers())) + { + if(CollectionUtils.nullSafeIsEmpty(qInstance.getSchedulableTypes())) + { + QScheduleManager.defineDefaultSchedulableTypesInInstance(qInstance); + } + } } @@ -1030,6 +1043,50 @@ public class QInstanceEnricher + /******************************************************************************* + ** Do a default mapping from an underscore_style field name to a camelCase name. + ** + ** Examples: + **
    + **
  • word_another_word_more_words -> wordAnotherWordMoreWords
    • + **
  • l_ul_ul_ul -> lUlUlUl
    • + **
  • tla_first -> tlaFirst
    • + **
  • word_then_tla_in_middle -> wordThenTlaInMiddle
    • + **
  • end_with_tla -> endWithTla
    • + **
  • tla_and_another_tla -> tlaAndAnotherTla
    • + **
  • ALL_CAPS -> allCaps
    • + **
+ *******************************************************************************/ + public static String inferNameFromBackendName(String backendName) + { + StringBuilder rs = new StringBuilder(); + + //////////////////////////////////////////////////////////////////////////////////////// + // build a list of words in the name, then join them with _ and lower-case the result // + //////////////////////////////////////////////////////////////////////////////////////// + String[] words = backendName.toLowerCase(Locale.ROOT).split("_"); + for(int i = 0; i < words.length; i++) + { + String word = words[i]; + if(i == 0) + { + rs.append(word); + } + else + { + rs.append(word.substring(0, 1).toUpperCase()); + if(word.length() > 1) + { + rs.append(word.substring(1)); + } + } + } + + return (rs.toString()); + } + + + /******************************************************************************* ** If a app didn't have any sections, generate "sensible defaults" *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index f16c4d7f..a30f1a1d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -35,6 +36,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TimeZone; import java.util.function.Supplier; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; @@ -104,6 +106,7 @@ import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda; +import org.quartz.CronExpression; /******************************************************************************* @@ -432,6 +435,11 @@ public class QInstanceValidator assertCondition(qInstance.getProcesses() != null && qInstance.getProcess(queue.getProcessName()) != null, "Unrecognized processName for queue: " + name); } + if(queue.getSchedule() != null) + { + validateScheduleMetaData(queue.getSchedule(), qInstance, "SQSQueueProvider " + name + ", schedule: "); + } + runPlugins(QQueueMetaData.class, queue, qInstance); }); } @@ -1013,6 +1021,11 @@ public class QInstanceValidator assertCondition(qInstance.getAutomationProvider(providerName) != null, " has an unrecognized providerName: " + providerName); } + if(automationDetails.getSchedule() != null) + { + validateScheduleMetaData(automationDetails.getSchedule(), qInstance, prefix + " automationDetails, schedule: "); + } + ////////////////////////////////// // validate the status tracking // ////////////////////////////////// @@ -1400,13 +1413,17 @@ public class QInstanceValidator if(process.getSchedule() != null) { QScheduleMetaData schedule = process.getSchedule(); - assertCondition(schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null, "Either repeat millis or repeat seconds must be set on schedule in process " + processName); + validateScheduleMetaData(schedule, qInstance, "Process " + processName + ", schedule: "); + } - if(schedule.getVariantBackend() != null) - { - assertCondition(qInstance.getBackend(schedule.getVariantBackend()) != null, "A variant backend was not found for " + schedule.getVariantBackend()); - assertCondition(schedule.getVariantRunStrategy() != null, "A variant run strategy was not set for " + schedule.getVariantBackend() + " on schedule in process " + processName); - } + if(process.getVariantBackend() != null) + { + assertCondition(qInstance.getBackend(process.getVariantBackend()) != null, "Process " + processName + ", a variant backend was not found named " + process.getVariantBackend()); + assertCondition(process.getVariantRunStrategy() != null, "A variant run strategy was not set for process " + processName + " (which does specify a variant backend)"); + } + else + { + assertCondition(process.getVariantRunStrategy() == null, "A variant run strategy was set for process " + processName + " (which isn't allowed, since it does not specify a variant backend)"); } for(QSupplementalProcessMetaData supplementalProcessMetaData : CollectionUtils.nonNullMap(process.getSupplementalMetaData()).values()) @@ -1421,6 +1438,50 @@ public class QInstanceValidator + /******************************************************************************* + ** + *******************************************************************************/ + private void validateScheduleMetaData(QScheduleMetaData schedule, QInstance qInstance, String prefix) + { + boolean isRepeat = schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null; + boolean isCron = StringUtils.hasContent(schedule.getCronExpression()); + assertCondition(isRepeat || isCron, prefix + " either repeatMillis or repeatSeconds or cronExpression must be set"); + assertCondition(!(isRepeat && isCron), prefix + " both a repeat time and cronExpression may not be set"); + + if(isCron) + { + boolean hasDelay = schedule.getInitialDelayMillis() != null || schedule.getInitialDelaySeconds() != null; + assertCondition(!hasDelay, prefix + " a cron schedule may not have an initial delay"); + + try + { + CronExpression.validateExpression(schedule.getCronExpression()); + } + catch(ParseException pe) + { + errors.add(prefix + " invalid cron expression: " + pe.getMessage()); + } + + if(assertCondition(StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a cron schedule must specify a cronTimeZoneId")) + { + String[] availableIDs = TimeZone.getAvailableIDs(); + Optional first = Arrays.stream(availableIDs).filter(id -> id.equals(schedule.getCronTimeZoneId())).findFirst(); + assertCondition(first.isPresent(), prefix + " unrecognized cronTimeZoneId: " + schedule.getCronTimeZoneId()); + } + } + else + { + assertCondition(!StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a non-cron schedule must not specify a cronTimeZoneId"); + } + + if(assertCondition(StringUtils.hasContent(schedule.getSchedulerName()), prefix + " is missing a scheduler name")) + { + assertCondition(qInstance.getScheduler(schedule.getSchedulerName()) != null, prefix + " is referencing an unknown scheduler name: " + schedule.getSchedulerName()); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java index ad96eac5..635a7410 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java @@ -48,6 +48,17 @@ public class CollectedLogMessage + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return "CollectedLogMessage{level=" + level + ", message='" + message + '\'' + ", exception=" + exception + '}'; + } + + + /******************************************************************************* ** Getter for message *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java index 42804602..cc361583 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java @@ -60,6 +60,11 @@ public interface QueryOrGetInputInterface QBackendTransaction getTransaction(); + /******************************************************************************* + ** + *******************************************************************************/ + String getTableName(); + /******************************************************************************* ** Setter for transaction *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java index 750fc91b..87acc4a0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.aggregate; import java.io.Serializable; import java.util.Objects; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; @@ -38,6 +39,17 @@ public class GroupBy implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + public GroupBy(QFieldMetaData field) + { + this.type = field.getType(); + this.fieldName = field.getName(); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java index da9dad45..bbee71d3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java @@ -23,10 +23,12 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.io.Serializable; +import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; /******************************************************************************* @@ -89,4 +91,18 @@ public class QueryOutput extends AbstractActionOutput implements Serializable return storage.getRecords(); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public List getRecordEntities(Class entityClass) throws QException + { + List rs = new ArrayList<>(); + for(QRecord record : storage.getRecords()) + { + rs.add(QRecordEntity.fromQRecord(entityClass, record)); + } + return (rs); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/common/TimeZonePossibleValueSourceMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/common/TimeZonePossibleValueSourceMetaDataProvider.java new file mode 100644 index 00000000..fa06e309 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/common/TimeZonePossibleValueSourceMetaDataProvider.java @@ -0,0 +1,78 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.common; + + +import java.util.ArrayList; +import java.util.List; +import java.util.TimeZone; +import java.util.function.Function; +import java.util.function.Predicate; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TimeZonePossibleValueSourceMetaDataProvider +{ + public static final String NAME = "timeZones"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QPossibleValueSource produce() + { + return (produce(null, null)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QPossibleValueSource produce(Predicate filter, Function labelMapper) + { + QPossibleValueSource possibleValueSource = new QPossibleValueSource() + .withName("timeZones") + .withType(QPossibleValueSourceType.ENUM) + .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY); + + List> enumValues = new ArrayList<>(); + for(String availableID : TimeZone.getAvailableIDs()) + { + if(filter == null || filter.test(availableID)) + { + String label = labelMapper == null ? availableID : labelMapper.apply(availableID); + enumValues.add(new QPossibleValue<>(availableID, label)); + } + } + + possibleValueSource.withEnumValues(enumValues); + return (possibleValueSource); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index 84a521f3..7c42e98f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -462,6 +462,16 @@ public class QRecord implements Serializable } + /******************************************************************************* + ** Getter for a single field's value + ** + *******************************************************************************/ + public Long getValueLong(String fieldName) + { + return (ValueUtils.getValueAsLong(values.get(fieldName))); + } + + /******************************************************************************* ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 115fcccc..255e4c7e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -53,8 +53,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import io.github.cdimascio.dotenv.Dotenv; @@ -91,6 +93,9 @@ public class QInstance private Map queueProviders = new LinkedHashMap<>(); private Map queues = new LinkedHashMap<>(); + private Map schedulers = new LinkedHashMap<>(); + private Map schedulableTypes = new LinkedHashMap<>(); + private Map supplementalMetaData = new LinkedHashMap<>(); private String deploymentMode; @@ -1224,4 +1229,106 @@ public class QInstance metaData.addSelfToInstance(this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addScheduler(QSchedulerMetaData scheduler) + { + String name = scheduler.getName(); + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a scheduler without a name.")); + } + if(this.schedulers.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second scheduler with name: " + name)); + } + this.schedulers.put(name, scheduler); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QSchedulerMetaData getScheduler(String name) + { + return (this.schedulers.get(name)); + } + + + + /******************************************************************************* + ** Getter for schedulers + ** + *******************************************************************************/ + public Map getSchedulers() + { + return schedulers; + } + + + + /******************************************************************************* + ** Setter for schedulers + ** + *******************************************************************************/ + public void setSchedulers(Map schedulers) + { + this.schedulers = schedulers; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addSchedulableType(SchedulableType schedulableType) + { + String name = schedulableType.getName(); + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a schedulableType without a name.")); + } + if(this.schedulableTypes.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second schedulableType with name: " + name)); + } + this.schedulableTypes.put(name, schedulableType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public SchedulableType getSchedulableType(String name) + { + return (this.schedulableTypes.get(name)); + } + + + + /******************************************************************************* + ** Getter for schedulableTypes + ** + *******************************************************************************/ + public Map getSchedulableTypes() + { + return schedulableTypes; + } + + + + /******************************************************************************* + ** Setter for schedulableTypes + ** + *******************************************************************************/ + public void setSchedulableTypes(Map schedulableTypes) + { + this.schedulableTypes = schedulableTypes; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/TopLevelMetaDataInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/TopLevelMetaDataInterface.java index 7be8db54..e3dc117f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/TopLevelMetaDataInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/TopLevelMetaDataInterface.java @@ -29,6 +29,11 @@ package com.kingsrook.qqq.backend.core.model.metadata; public interface TopLevelMetaDataInterface { + /******************************************************************************* + ** + *******************************************************************************/ + String getName(); + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java index 6dca9cde..31af5cff 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java @@ -24,7 +24,6 @@ package com.kingsrook.qqq.backend.core.model.metadata.automation; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; /******************************************************************************* @@ -35,8 +34,6 @@ public class QAutomationProviderMetaData implements TopLevelMetaDataInterface private String name; private QAutomationProviderType type; - private QScheduleMetaData schedule; - /******************************************************************************* @@ -107,40 +104,6 @@ public class QAutomationProviderMetaData implements TopLevelMetaDataInterface - /******************************************************************************* - ** Getter for schedule - ** - *******************************************************************************/ - public QScheduleMetaData getSchedule() - { - return schedule; - } - - - - /******************************************************************************* - ** Setter for schedule - ** - *******************************************************************************/ - public void setSchedule(QScheduleMetaData schedule) - { - this.schedule = schedule; - } - - - - /******************************************************************************* - ** Fluent setter for schedule - ** - *******************************************************************************/ - public QAutomationProviderMetaData withSchedule(QScheduleMetaData schedule) - { - this.schedule = schedule; - return (this); - } - - - /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java index 6e2d7541..6eb01b58 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java @@ -55,6 +55,17 @@ public class QBrandingMetaData implements TopLevelMetaDataInterface + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getName() + { + return "Branding"; + } + + + /******************************************************************************* ** Getter for companyName ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java index 8971a846..b0e8f8e7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java @@ -37,6 +37,7 @@ public enum QFieldType { STRING, INTEGER, + LONG, DECIMAL, BOOLEAN, DATE, @@ -65,6 +66,10 @@ public enum QFieldType { return (INTEGER); } + if(c.equals(Long.class) || c.equals(long.class)) + { + return (LONG); + } if(c.equals(BigDecimal.class)) { return (DECIMAL); @@ -110,7 +115,7 @@ public enum QFieldType *******************************************************************************/ public boolean isNumeric() { - return this == QFieldType.INTEGER || this == QFieldType.DECIMAL; + return this == QFieldType.INTEGER || this == QFieldType.LONG || this == QFieldType.DECIMAL; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java index 374650bf..4d5e5725 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.frontend; +import java.io.Serializable; import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonInclude; @@ -60,6 +61,7 @@ public class QFrontendWidgetMetaData protected Map icons; protected Map helpContent; + protected Map defaultValues; private final boolean hasPermission; @@ -95,6 +97,7 @@ public class QFrontendWidgetMetaData } this.helpContent = widgetMetaData.getHelpContent(); + this.defaultValues = widgetMetaData.getDefaultValues(); hasPermission = PermissionsHelper.hasWidgetPermission(actionInput, name); } @@ -274,4 +277,16 @@ public class QFrontendWidgetMetaData { return helpContent; } + + + + /******************************************************************************* + ** Getter for defaultValues + ** + *******************************************************************************/ + public Map getDefaultValues() + { + return defaultValues; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java index 946fa448..4db07304 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java @@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.layout; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -363,11 +365,11 @@ public class QAppMetaData implements QAppChildMetaData, MetaDataWithPermissionRu /******************************************************************************* ** *******************************************************************************/ - public QAppMetaData withSectionOfChildren(QAppSection section, QAppChildMetaData... children) + public QAppMetaData withSectionOfChildren(QAppSection section, Collection children) { this.addSection(section); - for(QAppChildMetaData child : children) + for(QAppChildMetaData child : CollectionUtils.nonNullCollection(children)) { withChild(child); if(child instanceof QTableMetaData) @@ -392,6 +394,15 @@ public class QAppMetaData implements QAppChildMetaData, MetaDataWithPermissionRu } + /******************************************************************************* + ** + *******************************************************************************/ + public QAppMetaData withSectionOfChildren(QAppSection section, QAppChildMetaData... children) + { + return (withSectionOfChildren(section, children == null ? null : Arrays.stream(children).toList())); + } + + /******************************************************************************* ** Getter for permissionRules diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java index 5f60e4b0..e14df8bc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java @@ -155,4 +155,26 @@ public class AbstractProcessMetaDataBuilder return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public AbstractProcessMetaDataBuilder withVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + processMetaData.setVariantRunStrategy(variantRunStrategy); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public AbstractProcessMetaDataBuilder withVariantBackend(String variantBackend) + { + processMetaData.setVariantBackend(variantBackend); + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index 8a1a5eae..fae291e8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -64,6 +64,9 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi private QScheduleMetaData schedule; + private VariantRunStrategy variantRunStrategy; + private String variantBackend; + private Map supplementalMetaData; @@ -671,4 +674,66 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi return (this); } + + /******************************************************************************* + ** Getter for variantRunStrategy + *******************************************************************************/ + public VariantRunStrategy getVariantRunStrategy() + { + return (this.variantRunStrategy); + } + + + + /******************************************************************************* + ** Setter for variantRunStrategy + *******************************************************************************/ + public void setVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + this.variantRunStrategy = variantRunStrategy; + } + + + + /******************************************************************************* + ** Fluent setter for variantRunStrategy + *******************************************************************************/ + public QProcessMetaData withVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + this.variantRunStrategy = variantRunStrategy; + return (this); + } + + + + /******************************************************************************* + ** Getter for variantBackend + *******************************************************************************/ + public String getVariantBackend() + { + return (this.variantBackend); + } + + + + /******************************************************************************* + ** Setter for variantBackend + *******************************************************************************/ + public void setVariantBackend(String variantBackend) + { + this.variantBackend = variantBackend; + } + + + + /******************************************************************************* + ** Fluent setter for variantBackend + *******************************************************************************/ + public QProcessMetaData withVariantBackend(String variantBackend) + { + this.variantBackend = variantBackend; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/VariantRunStrategy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/VariantRunStrategy.java new file mode 100644 index 00000000..888cc3ae --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/VariantRunStrategy.java @@ -0,0 +1,32 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.processes; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum VariantRunStrategy +{ + PARALLEL, + SERIAL +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java index 3825b9ff..6184db3f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java @@ -22,9 +22,6 @@ package com.kingsrook.qqq.backend.core.model.metadata.queues; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; - - /******************************************************************************* ** Meta-data for an source of Amazon SQS queues (e.g, an aws account/credential ** set, with a common base URL). @@ -39,8 +36,6 @@ public class SQSQueueProviderMetaData extends QQueueProviderMetaData private String region; private String baseURL; - private QScheduleMetaData schedule; - /******************************************************************************* @@ -201,38 +196,4 @@ public class SQSQueueProviderMetaData extends QQueueProviderMetaData return (this); } - - - /******************************************************************************* - ** Getter for schedule - ** - *******************************************************************************/ - public QScheduleMetaData getSchedule() - { - return schedule; - } - - - - /******************************************************************************* - ** Setter for schedule - ** - *******************************************************************************/ - public void setSchedule(QScheduleMetaData schedule) - { - this.schedule = schedule; - } - - - - /******************************************************************************* - ** Fluent setter for schedule - ** - *******************************************************************************/ - public SQSQueueProviderMetaData withSchedule(QScheduleMetaData schedule) - { - this.schedule = schedule; - return (this); - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java index ece9019a..d474a013 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java @@ -22,29 +22,41 @@ package com.kingsrook.qqq.backend.core.model.metadata.scheduleing; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + /******************************************************************************* ** Meta-data to define scheduled actions within QQQ. ** - ** Initially, only supports repeating jobs, either on a given # of seconds or millis. + ** Supports repeating jobs, either on a given # of seconds or millis, or cron + ** expressions (though cron may not be supported by all schedulers!) + ** ** Can also specify an initialDelay - e.g., to avoid all jobs starting up at the ** same moment. ** - ** In the future we most likely would want to allow cron strings to be added here. *******************************************************************************/ public class QScheduleMetaData { - public enum RunStrategy - {PARALLEL, SERIAL} - - + private String schedulerName; + private String description; private Integer repeatSeconds; private Integer repeatMillis; private Integer initialDelaySeconds; private Integer initialDelayMillis; - private RunStrategy variantRunStrategy; - private String variantBackend; + private String cronExpression; + private String cronTimeZoneId; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean isCron() + { + return StringUtils.hasContent(cronExpression); + } @@ -185,63 +197,125 @@ public class QScheduleMetaData /******************************************************************************* - ** Getter for variantBackend + ** Getter for cronExpression *******************************************************************************/ - public String getVariantBackend() + public String getCronExpression() { - return (this.variantBackend); + return (this.cronExpression); } /******************************************************************************* - ** Setter for variantBackend + ** Setter for cronExpression *******************************************************************************/ - public void setVariantBackend(String variantBackend) + public void setCronExpression(String cronExpression) { - this.variantBackend = variantBackend; + this.cronExpression = cronExpression; } /******************************************************************************* - ** Fluent setter for variantBackend + ** Fluent setter for cronExpression *******************************************************************************/ - public QScheduleMetaData withBackendVariant(String backendVariant) + public QScheduleMetaData withCronExpression(String cronExpression) { - this.variantBackend = backendVariant; + this.cronExpression = cronExpression; return (this); } /******************************************************************************* - ** Getter for variantRunStrategy + ** Getter for cronTimeZoneId *******************************************************************************/ - public RunStrategy getVariantRunStrategy() + public String getCronTimeZoneId() { - return (this.variantRunStrategy); + return (this.cronTimeZoneId); } /******************************************************************************* - ** Setter for variantRunStrategy + ** Setter for cronTimeZoneId *******************************************************************************/ - public void setVariantRunStrategy(RunStrategy variantRunStrategy) + public void setCronTimeZoneId(String cronTimeZoneId) { - this.variantRunStrategy = variantRunStrategy; + this.cronTimeZoneId = cronTimeZoneId; } /******************************************************************************* - ** Fluent setter for variantRunStrategy + ** Fluent setter for cronTimeZoneId *******************************************************************************/ - public QScheduleMetaData withVariantRunStrategy(RunStrategy variantRunStrategy) + public QScheduleMetaData withCronTimeZoneId(String cronTimeZoneId) { - this.variantRunStrategy = variantRunStrategy; + this.cronTimeZoneId = cronTimeZoneId; return (this); } + + + /******************************************************************************* + ** Getter for schedulerName + *******************************************************************************/ + public String getSchedulerName() + { + return (this.schedulerName); + } + + + + /******************************************************************************* + ** Setter for schedulerName + *******************************************************************************/ + public void setSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + } + + + + /******************************************************************************* + ** Fluent setter for schedulerName + *******************************************************************************/ + public QScheduleMetaData withSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + return (this); + } + + + /******************************************************************************* + ** Getter for description + *******************************************************************************/ + public String getDescription() + { + return (this.description); + } + + + + /******************************************************************************* + ** Setter for description + *******************************************************************************/ + public void setDescription(String description) + { + this.description = description; + } + + + + /******************************************************************************* + ** Fluent setter for description + *******************************************************************************/ + public QScheduleMetaData withDescription(String description) + { + this.description = description; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java new file mode 100644 index 00000000..9eda36b2 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java @@ -0,0 +1,141 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.scheduleing; + + +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class QSchedulerMetaData implements TopLevelMetaDataInterface +{ + private String name; + private String type; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean supportsCronSchedules() + { + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean mayUseInScheduledJobsTable() + { + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public abstract QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier systemSessionSupplier) throws QException; + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public QSchedulerMetaData withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for type + *******************************************************************************/ + public String getType() + { + return (this.type); + } + + + + /******************************************************************************* + ** Setter for type + *******************************************************************************/ + public void setType(String type) + { + this.type = type; + } + + + + /******************************************************************************* + ** Fluent setter for type + *******************************************************************************/ + public QSchedulerMetaData withType(String type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addSelfToInstance(QInstance qInstance) + { + qInstance.addScheduler(this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java new file mode 100644 index 00000000..3367cd25 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java @@ -0,0 +1,130 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz; + + +import java.util.Properties; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuartzSchedulerMetaData extends QSchedulerMetaData +{ + private static final QLogger LOG = QLogger.getLogger(QuartzSchedulerMetaData.class); + + public static final String TYPE = "quartz"; + + private Properties properties; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public QuartzSchedulerMetaData() + { + setType(TYPE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean supportsCronSchedules() + { + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean mayUseInScheduledJobsTable() + { + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier systemSessionSupplier) throws QException + { + try + { + QuartzScheduler quartzScheduler = QuartzScheduler.initInstance(qInstance, getName(), getProperties(), systemSessionSupplier); + return (quartzScheduler); + } + catch(Exception e) + { + LOG.error("Error initializing quartz scheduler", e); + throw (new QException("Error initializing quartz scheduler", e)); + } + } + + + + /******************************************************************************* + ** Getter for properties + *******************************************************************************/ + public Properties getProperties() + { + return (this.properties); + } + + + + /******************************************************************************* + ** Setter for properties + *******************************************************************************/ + public void setProperties(Properties properties) + { + this.properties = properties; + } + + + + /******************************************************************************* + ** Fluent setter for properties + *******************************************************************************/ + public QuartzSchedulerMetaData withProperties(Properties properties) + { + this.properties = properties; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java new file mode 100644 index 00000000..69882b41 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple; + + +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.simple.SimpleScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SimpleSchedulerMetaData extends QSchedulerMetaData +{ + public static final String TYPE = "simple"; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SimpleSchedulerMetaData() + { + setType(TYPE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean supportsCronSchedules() + { + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean mayUseInScheduledJobsTable() + { + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier systemSessionSupplier) + { + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); + simpleScheduler.setSessionSupplier(systemSessionSupplier); + simpleScheduler.setSchedulerName(getName()); + return simpleScheduler; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java index e5e39c73..f272620f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java @@ -22,6 +22,10 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables; +import java.util.HashSet; +import java.util.Set; + + /******************************************************************************* ** Things that can be done to tables, fields. ** @@ -38,5 +42,26 @@ public enum Capability // keep these values in sync with Capability.ts in qqq-frontend-core // /////////////////////////////////////////////////////////////////////// - QUERY_STATS + QUERY_STATS; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Set allReadCapabilities() + { + return (new HashSet<>(Set.of(TABLE_QUERY, TABLE_GET, TABLE_COUNT, QUERY_STATS))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Set allWriteCapabilities() + { + return (new HashSet<>(Set.of(TABLE_INSERT, TABLE_UPDATE, TABLE_DELETE))); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java index 303e2932..13a57662 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables.automation; import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; /******************************************************************************* @@ -37,6 +38,8 @@ public class QTableAutomationDetails private Integer overrideBatchSize; + private QScheduleMetaData schedule; + private String shardByFieldName; // field in "this" table, to use for sharding private String shardSourceTableName; // name of the table where the shards are defined as rows private String shardLabelFieldName; // field in shard-source-table to use for labeling shards @@ -317,4 +320,35 @@ public class QTableAutomationDetails return (this); } + + /******************************************************************************* + ** Getter for schedule + *******************************************************************************/ + public QScheduleMetaData getSchedule() + { + return (this.schedule); + } + + + + /******************************************************************************* + ** Setter for schedule + *******************************************************************************/ + public void setSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + } + + + + /******************************************************************************* + ** Fluent setter for schedule + *******************************************************************************/ + public QTableAutomationDetails withSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java index 62e460bf..c9cbd826 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java @@ -65,6 +65,7 @@ public class QueryStatMetaDataProvider instance.addTable(defineStandardTable(QueryStatJoinTable.TABLE_NAME, QueryStatJoinTable.class, backendName, backendDetailEnricher)); instance.addTable(defineStandardTable(QueryStatCriteriaField.TABLE_NAME, QueryStatCriteriaField.class, backendName, backendDetailEnricher) + .withIcon(new QIcon().withName("filter_alt")) .withExposedJoin(new ExposedJoin().withJoinTable(QueryStat.TABLE_NAME)) ); @@ -115,6 +116,7 @@ public class QueryStatMetaDataProvider QTableMetaData table = new QTableMetaData() .withName(QueryStat.TABLE_NAME) + .withIcon(new QIcon().withName("query_stats")) .withBackendName(backendName) .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE)) .withRecordLabelFormat("%s") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java new file mode 100644 index 00000000..e923d549 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java @@ -0,0 +1,496 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.common.TimeZonePossibleValueSourceMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.data.QAssociation; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MutableMap; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJob extends QRecordEntity +{ + public static final String TABLE_NAME = "scheduledJob"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(isRequired = true, maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE) + private String label; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS) + private String description; + + @QField(isRequired = true, label = "Scheduler", maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = SchedulersPossibleValueSource.NAME) + private String schedulerName; + + @QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String cronExpression; + + @QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = TimeZonePossibleValueSourceMetaDataProvider.NAME) + private String cronTimeZoneId; + + @QField(displayFormat = DisplayFormat.COMMAS) + private Integer repeatSeconds; + + @QField(isRequired = true, maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = ScheduledJobTypePossibleValueSource.NAME) + private String type; + + @QField(isRequired = true) + private Boolean isActive; + + @QAssociation(name = ScheduledJobParameter.TABLE_NAME) + private List jobParameters; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledJob() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledJob(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public ScheduledJob withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public ScheduledJob withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public ScheduledJob withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public ScheduledJob withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for description + *******************************************************************************/ + public String getDescription() + { + return (this.description); + } + + + + /******************************************************************************* + ** Setter for description + *******************************************************************************/ + public void setDescription(String description) + { + this.description = description; + } + + + + /******************************************************************************* + ** Fluent setter for description + *******************************************************************************/ + public ScheduledJob withDescription(String description) + { + this.description = description; + return (this); + } + + + + /******************************************************************************* + ** Getter for cronExpression + *******************************************************************************/ + public String getCronExpression() + { + return (this.cronExpression); + } + + + + /******************************************************************************* + ** Setter for cronExpression + *******************************************************************************/ + public void setCronExpression(String cronExpression) + { + this.cronExpression = cronExpression; + } + + + + /******************************************************************************* + ** Fluent setter for cronExpression + *******************************************************************************/ + public ScheduledJob withCronExpression(String cronExpression) + { + this.cronExpression = cronExpression; + return (this); + } + + + + /******************************************************************************* + ** Getter for cronTimeZoneId + *******************************************************************************/ + public String getCronTimeZoneId() + { + return (this.cronTimeZoneId); + } + + + + /******************************************************************************* + ** Setter for cronTimeZoneId + *******************************************************************************/ + public void setCronTimeZoneId(String cronTimeZoneId) + { + this.cronTimeZoneId = cronTimeZoneId; + } + + + + /******************************************************************************* + ** Fluent setter for cronTimeZoneId + *******************************************************************************/ + public ScheduledJob withCronTimeZoneId(String cronTimeZoneId) + { + this.cronTimeZoneId = cronTimeZoneId; + return (this); + } + + + + /******************************************************************************* + ** Getter for isActive + *******************************************************************************/ + public Boolean getIsActive() + { + return (this.isActive); + } + + + + /******************************************************************************* + ** Setter for isActive + *******************************************************************************/ + public void setIsActive(Boolean isActive) + { + this.isActive = isActive; + } + + + + /******************************************************************************* + ** Fluent setter for isActive + *******************************************************************************/ + public ScheduledJob withIsActive(Boolean isActive) + { + this.isActive = isActive; + return (this); + } + + + + /******************************************************************************* + ** Getter for schedulerName + *******************************************************************************/ + public String getSchedulerName() + { + return (this.schedulerName); + } + + + + /******************************************************************************* + ** Setter for schedulerName + *******************************************************************************/ + public void setSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + } + + + + /******************************************************************************* + ** Fluent setter for schedulerName + *******************************************************************************/ + public ScheduledJob withSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + return (this); + } + + + + /******************************************************************************* + ** Getter for type + *******************************************************************************/ + public String getType() + { + return (this.type); + } + + + + /******************************************************************************* + ** Setter for type + *******************************************************************************/ + public void setType(String type) + { + this.type = type; + } + + + + /******************************************************************************* + ** Fluent setter for type + *******************************************************************************/ + public ScheduledJob withType(String type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** Getter for jobParameters + *******************************************************************************/ + public List getJobParameters() + { + return (this.jobParameters); + } + + + + /******************************************************************************* + ** Getter for jobParameters - but a map of just the key=value pairs. + *******************************************************************************/ + public Map getJobParametersMap() + { + if(CollectionUtils.nullSafeIsEmpty(this.jobParameters)) + { + return (new HashMap<>()); + } + + /////////////////////////////////////////////////////////////////////////////////////// + // wrap in mutable map, just to avoid any immutable or other bs from toMap's default // + /////////////////////////////////////////////////////////////////////////////////////// + return new MutableMap<>(jobParameters.stream().collect(Collectors.toMap(ScheduledJobParameter::getKey, ScheduledJobParameter::getValue))); + } + + + + /******************************************************************************* + ** Setter for jobParameters + *******************************************************************************/ + public void setJobParameters(List jobParameters) + { + this.jobParameters = jobParameters; + } + + + + /******************************************************************************* + ** Fluent setter for jobParameters + *******************************************************************************/ + public ScheduledJob withJobParameters(List jobParameters) + { + this.jobParameters = jobParameters; + return (this); + } + + + /******************************************************************************* + ** Getter for repeatSeconds + *******************************************************************************/ + public Integer getRepeatSeconds() + { + return (this.repeatSeconds); + } + + + + /******************************************************************************* + ** Setter for repeatSeconds + *******************************************************************************/ + public void setRepeatSeconds(Integer repeatSeconds) + { + this.repeatSeconds = repeatSeconds; + } + + + + /******************************************************************************* + ** Fluent setter for repeatSeconds + *******************************************************************************/ + public ScheduledJob withRepeatSeconds(Integer repeatSeconds) + { + this.repeatSeconds = repeatSeconds; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobParameter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobParameter.java new file mode 100644 index 00000000..ccfa816e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobParameter.java @@ -0,0 +1,266 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobParameter extends QRecordEntity +{ + public static final String TABLE_NAME = "scheduledJobParameter"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(possibleValueSourceName = ScheduledJob.TABLE_NAME, isRequired = true) + private Integer scheduledJobId; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, isRequired = true) + private String key; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String value; + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledJobParameter() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledJobParameter(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public ScheduledJobParameter withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public ScheduledJobParameter withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public ScheduledJobParameter withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for scheduledJobId + *******************************************************************************/ + public Integer getScheduledJobId() + { + return (this.scheduledJobId); + } + + + + /******************************************************************************* + ** Setter for scheduledJobId + *******************************************************************************/ + public void setScheduledJobId(Integer scheduledJobId) + { + this.scheduledJobId = scheduledJobId; + } + + + + /******************************************************************************* + ** Fluent setter for scheduledJobId + *******************************************************************************/ + public ScheduledJobParameter withScheduledJobId(Integer scheduledJobId) + { + this.scheduledJobId = scheduledJobId; + return (this); + } + + + + /******************************************************************************* + ** Getter for key + *******************************************************************************/ + public String getKey() + { + return (this.key); + } + + + + /******************************************************************************* + ** Setter for key + *******************************************************************************/ + public void setKey(String key) + { + this.key = key; + } + + + + /******************************************************************************* + ** Fluent setter for key + *******************************************************************************/ + public ScheduledJobParameter withKey(String key) + { + this.key = key; + return (this); + } + + + + /******************************************************************************* + ** Getter for value + *******************************************************************************/ + public String getValue() + { + return (this.value); + } + + + + /******************************************************************************* + ** Setter for value + *******************************************************************************/ + public void setValue(String value) + { + this.value = value; + } + + + + /******************************************************************************* + ** Fluent setter for value + *******************************************************************************/ + public ScheduledJobParameter withValue(String value) + { + this.value = value; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java new file mode 100644 index 00000000..56114667 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java @@ -0,0 +1,37 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +/******************************************************************************* + ** enum of core schedulable types that QQQ schedule manager directly knows about. + ** + ** note though, that applications can define their own schedulable types, + ** by adding SchedulableType meta-data to the QInstance, and providing classes + ** that implement SchedulableRunner. + *******************************************************************************/ +public enum ScheduledJobType +{ + PROCESS, + QUEUE_PROCESSOR, + TABLE_AUTOMATIONS +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobTypePossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobTypePossibleValueSource.java new file mode 100644 index 00000000..43d26dd5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobTypePossibleValueSource.java @@ -0,0 +1,87 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobTypePossibleValueSource implements QCustomPossibleValueProvider +{ + public static final String NAME = "scheduledJobType"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QPossibleValue getPossibleValue(Serializable idValue) + { + SchedulableType schedulableType = QContext.getQInstance().getSchedulableType(String.valueOf(idValue)); + if(schedulableType != null) + { + return schedulableTypeToPossibleValue(schedulableType); + } + + return null; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List> search(SearchPossibleValueSourceInput input) throws QException + { + List> rs = new ArrayList<>(); + for(SchedulableType schedulableType : CollectionUtils.nonNullMap(QContext.getQInstance().getSchedulableTypes()).values()) + { + rs.add(schedulableTypeToPossibleValue(schedulableType)); + } + return rs; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QPossibleValue schedulableTypeToPossibleValue(SchedulableType schedulableType) + { + return new QPossibleValue<>(schedulableType.getName(), schedulableType.getName()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java new file mode 100644 index 00000000..f814ff88 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java @@ -0,0 +1,249 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers.ScheduledJobParameterTableCustomizer; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers.ScheduledJobTableCustomizer; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobsMetaDataProvider +{ + private static final String JOB_PARAMETER_JOIN_NAME = QJoinMetaData.makeInferredJoinName(ScheduledJob.TABLE_NAME, ScheduledJobParameter.TABLE_NAME); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineAll(QInstance instance, String backendName, Consumer backendDetailEnricher) throws QException + { + defineStandardTables(instance, backendName, backendDetailEnricher); + instance.addPossibleValueSource(QPossibleValueSource.newForTable(ScheduledJob.TABLE_NAME)); + instance.addPossibleValueSource(defineScheduledJobTypePossibleValueSource()); + instance.addPossibleValueSource(defineSchedulersPossibleValueSource()); + defineStandardJoins(instance); + defineStandardWidgets(instance); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineStandardWidgets(QInstance instance) + { + QJoinMetaData join = instance.getJoin(JOB_PARAMETER_JOIN_NAME); + instance.addWidget(ChildRecordListRenderer.widgetMetaDataBuilder(join) + .withCanAddChildRecord(true) + .withManageAssociationName(ScheduledJobParameter.TABLE_NAME) + .withLabel("Parameters") + .getWidgetMetaData() + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineStandardJoins(QInstance instance) + { + instance.addJoin(new QJoinMetaData() + .withType(JoinType.ONE_TO_MANY) + .withLeftTable(ScheduledJob.TABLE_NAME) + .withRightTable(ScheduledJobParameter.TABLE_NAME) + .withJoinOn(new JoinOn("id", "scheduledJobId")) + .withOrderBy(new QFilterOrderBy("id")) + .withInferredName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineStandardTables(QInstance instance, String backendName, Consumer backendDetailEnricher) throws QException + { + for(QTableMetaData tableMetaData : defineStandardTables(backendName, backendDetailEnricher)) + { + instance.addTable(tableMetaData); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List defineStandardTables(String backendName, Consumer backendDetailEnricher) throws QException + { + List rs = new ArrayList<>(); + rs.add(enrich(backendDetailEnricher, defineScheduledJobTable(backendName))); + rs.add(enrich(backendDetailEnricher, defineScheduledJobParameterTable(backendName))); + return (rs); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData enrich(Consumer backendDetailEnricher, QTableMetaData table) + { + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineStandardTable(String backendName, String name, Class fieldsFromEntity) throws QException + { + return new QTableMetaData() + .withName(name) + .withBackendName(backendName) + .withPrimaryKeyField("id") + .withFieldsFromEntity(fieldsFromEntity); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineScheduledJobTable(String backendName) throws QException + { + QTableMetaData tableMetaData = defineStandardTable(backendName, ScheduledJob.TABLE_NAME, ScheduledJob.class) + .withRecordLabelFormat("%s") + .withRecordLabelFields("label") + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "description"))) + .withSection(new QFieldSection("schedule", new QIcon().withName("alarm"), Tier.T2, List.of("cronExpression", "cronTimeZoneId", "repeatSeconds"))) + .withSection(new QFieldSection("settings", new QIcon().withName("tune"), Tier.T2, List.of("type", "isActive", "schedulerName"))) + .withSection(new QFieldSection("parameters", new QIcon().withName("list"), Tier.T2).withWidgetName(JOB_PARAMETER_JOIN_NAME)) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + QCodeReference customizerReference = new QCodeReference(ScheduledJobTableCustomizer.class); + tableMetaData.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_INSERT_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_DELETE_RECORD, customizerReference); + + tableMetaData.withAssociation(new Association() + .withName(ScheduledJobParameter.TABLE_NAME) + .withAssociatedTableName(ScheduledJobParameter.TABLE_NAME) + .withJoinName(JOB_PARAMETER_JOIN_NAME)); + + tableMetaData.withExposedJoin(new ExposedJoin() + .withJoinTable(ScheduledJobParameter.TABLE_NAME) + .withJoinPath(List.of(JOB_PARAMETER_JOIN_NAME)) + .withLabel("Parameters")); + + return (tableMetaData); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineScheduledJobParameterTable(String backendName) throws QException + { + QTableMetaData tableMetaData = defineStandardTable(backendName, ScheduledJobParameter.TABLE_NAME, ScheduledJobParameter.class) + .withRecordLabelFormat("%s - %s") + .withRecordLabelFields("scheduledJobId", "key") + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "scheduledJobId", "key", "value"))) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + + QCodeReference customizerReference = new QCodeReference(ScheduledJobParameterTableCustomizer.class); + tableMetaData.withCustomizer(TableCustomizers.POST_INSERT_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_DELETE_RECORD, customizerReference); + + tableMetaData.withExposedJoin(new ExposedJoin() + .withJoinTable(ScheduledJob.TABLE_NAME) + .withJoinPath(List.of(JOB_PARAMETER_JOIN_NAME)) + .withLabel("Scheduled Job")); + + return (tableMetaData); + } + + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QPossibleValueSource defineScheduledJobTypePossibleValueSource() + { + return (new QPossibleValueSource() + .withName(ScheduledJobTypePossibleValueSource.NAME) + .withType(QPossibleValueSourceType.CUSTOM) + .withCustomCodeReference(new QCodeReference(ScheduledJobTypePossibleValueSource.class))); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + private QPossibleValueSource defineSchedulersPossibleValueSource() + { + return (new QPossibleValueSource() + .withName(SchedulersPossibleValueSource.NAME) + .withType(QPossibleValueSourceType.CUSTOM) + .withCustomCodeReference(new QCodeReference(SchedulersPossibleValueSource.class))); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java new file mode 100644 index 00000000..e46c81c0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SchedulersPossibleValueSource implements QCustomPossibleValueProvider +{ + public static final String NAME = "schedulers"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QPossibleValue getPossibleValue(Serializable idValue) + { + QSchedulerMetaData scheduler = QContext.getQInstance().getScheduler(String.valueOf(idValue)); + if(scheduler != null) + { + return schedulerToPossibleValue(scheduler); + } + + return null; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List> search(SearchPossibleValueSourceInput input) throws QException + { + List> rs = new ArrayList<>(); + for(QSchedulerMetaData scheduler : CollectionUtils.nonNullMap(QContext.getQInstance().getSchedulers()).values()) + { + if(scheduler.mayUseInScheduledJobsTable()) + { + rs.add(schedulerToPossibleValue(scheduler)); + } + } + return rs; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QPossibleValue schedulerToPossibleValue(QSchedulerMetaData scheduler) + { + return new QPossibleValue<>(scheduler.getName(), scheduler.getName()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobParameterTableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobParameterTableCustomizer.java new file mode 100644 index 00000000..798efdba --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobParameterTableCustomizer.java @@ -0,0 +1,215 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers; + + +import java.time.Instant; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobParameter; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobParameterTableCustomizer implements TableCustomizerInterface +{ + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + /////////////////////////////////////////////////////////////////////////////////////// + // if we're in this insert as a result of an insert (or update) on a different table // + // (e.g., under a manageAssociations call), then return with noop - assume that the // + // parent table's customizer will do what needed to be done. // + /////////////////////////////////////////////////////////////////////////////////////// + if(!isThisAnActionDirectlyOnThisTable()) + { + return (records); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else - this was an action directly on this table - so bump all of the parent records, to get them rescheduled // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + bumpParentRecords(records, Optional.empty()); + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void bumpParentRecords(List records, Optional> oldRecordList) throws QException + { + try + { + /////////////////////////////////////////////////////////////////////////////////////////// + // (listing) hash up the records by scheduledJobId - we'll use this to have a set of the // + // job ids, and in case we need to add warnings to them later // + /////////////////////////////////////////////////////////////////////////////////////////// + ListingHash recordsByJobId = new ListingHash<>(); + for(QRecord record : records) + { + recordsByJobId.add(record.getValueInteger("scheduledJobId"), record); + } + + Set scheduledJobIds = new HashSet<>(recordsByJobId.keySet()); + + //////////////////////////////////////////////////////////////////////////////// + // if we have an old record list (e.g., is an edit), add any job ids that are // + // in those too, e.g., in case moving a param from one job to another... // + // note, we won't line these up for doing a proper warning on these... // + //////////////////////////////////////////////////////////////////////////////// + if(oldRecordList.isPresent()) + { + for(QRecord oldRecord : oldRecordList.get()) + { + scheduledJobIds.add(oldRecord.getValueInteger("scheduledJobId")); + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // update the modify date on the scheduled jobs - to get their post-actions to run, to reschedule // + //////////////////////////////////////////////////////////////////////////////////////////////////// + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(ScheduledJob.TABLE_NAME); + updateInput.setRecords(scheduledJobIds.stream() + .map(id -> new QRecord().withValue("id", id).withValue("modifyDate", Instant.now())) + .toList()); + UpdateOutput updateOutput = new UpdateAction().execute(updateInput); + + //////////////////////////////////////////////////////////////////////////////////////// + // look for warnings on those jobs - and propagate them to the params we just stored. // + //////////////////////////////////////////////////////////////////////////////////////// + for(QRecord updatedScheduledJob : updateOutput.getRecords()) + { + if(CollectionUtils.nullSafeHasContents(updatedScheduledJob.getWarnings())) + { + for(QRecord paramToWarn : CollectionUtils.nonNullList(recordsByJobId.get(updatedScheduledJob.getValueInteger("id")))) + { + paramToWarn.setWarnings(updatedScheduledJob.getWarnings()); + } + } + } + } + catch(Exception e) + { + LOG.warn("Error in scheduledJobParameter post-crud", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + ///////////////////////////////////////////////////////////////////////////// + // if we're in this update as a result of an update on a different table // + // (e.g., under a manageAssociations call), then return with noop - assume // + // that the parent table's customizer will do what needed to be done. // + ///////////////////////////////////////////////////////////////////////////// + if(!isThisAnActionDirectlyOnThisTable()) + { + return (records); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else - this was an action directly on this table - so bump all of the parent records, to get them rescheduled // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + bumpParentRecords(records, oldRecordList); + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postDelete(DeleteInput deleteInput, List records) throws QException + { + ///////////////////////////////////////////////////////////////////////////// + // if we're in this update as a result of an update on a different table // + // (e.g., under a manageAssociations call), then return with noop - assume // + // that the parent table's customizer will do what needed to be done. // + ///////////////////////////////////////////////////////////////////////////// + if(!isThisAnActionDirectlyOnThisTable()) + { + return (records); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else - this was an action directly on this table - so bump all of the parent records, to get them rescheduled // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + bumpParentRecords(records, Optional.empty()); + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean isThisAnActionDirectlyOnThisTable() + { + Optional firstActionInStack = QContext.getFirstActionInStack(); + if(firstActionInStack.isPresent()) + { + if(firstActionInStack.get() instanceof AbstractTableActionInput tableActionInput) + { + if(!ScheduledJobParameter.TABLE_NAME.equals(tableActionInput.getTableName())) + { + return (false); + } + } + } + return (true); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java new file mode 100644 index 00000000..7203f500 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java @@ -0,0 +1,315 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers; + + +import java.text.ParseException; +import java.util.Collections; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.quartz.CronScheduleBuilder; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobTableCustomizer implements TableCustomizerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + validateConditionalFields(records, Collections.emptyMap()); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + scheduleJobsForRecordList(records); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + Map freshOldRecordsWithAssociationsMap = CollectionUtils.recordsToMap(freshlyQueryForRecordsWithAssociations(oldRecordList.get()), "id", Integer.class); + + validateConditionalFields(records, freshOldRecordsWithAssociationsMap); + + if(isPreview || oldRecordList.isEmpty()) + { + return (records); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // refresh the old-records w/ versions that have associations - so we can use those in the post-update to property unschedule things // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ListIterator iterator = oldRecordList.get().listIterator(); + while(iterator.hasNext()) + { + QRecord record = iterator.next(); + QRecord freshRecord = freshOldRecordsWithAssociationsMap.get(record.getValue("id")); + if(freshRecord != null) + { + iterator.set(freshRecord); + } + } + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void validateConditionalFields(List records, Map freshOldRecordsWithAssociationsMap) + { + QRecord blankRecord = new QRecord(); + for(QRecord record : records) + { + QRecord oldRecord = Objects.requireNonNullElse(freshOldRecordsWithAssociationsMap.get(record.getValueInteger("id")), blankRecord); + String cronExpression = record.getValues().containsKey("cronExpression") ? record.getValueString("cronExpression") : oldRecord.getValueString("cronExpression"); + String cronTimeZoneId = record.getValues().containsKey("cronTimeZoneId") ? record.getValueString("cronTimeZoneId") : oldRecord.getValueString("cronTimeZoneId"); + String repeatSeconds = record.getValues().containsKey("repeatSeconds") ? record.getValueString("repeatSeconds") : oldRecord.getValueString("repeatSeconds"); + + if(StringUtils.hasContent(cronExpression)) + { + if(StringUtils.hasContent(repeatSeconds)) + { + record.addError(new BadInputStatusMessage("Cron Expression and Repeat Seconds may not both be given.")); + } + + try + { + CronScheduleBuilder.cronScheduleNonvalidatedExpression(cronExpression); + } + catch(ParseException e) + { + record.addError(new BadInputStatusMessage("Cron Expression [" + cronExpression + "] is not valid: " + e.getMessage())); + } + + if(!StringUtils.hasContent(cronTimeZoneId)) + { + record.addError(new BadInputStatusMessage("If a Cron Expression is given, then a Cron Time Zone is required.")); + } + } + else + { + if(!StringUtils.hasContent(repeatSeconds)) + { + record.addError(new BadInputStatusMessage("Either Cron Expression or Repeat Seconds must be given.")); + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + if(oldRecordList.isPresent()) + { + Set idsWithErrors = getRecordIdsWithErrors(records); + unscheduleJobsForRecordList(oldRecordList.get(), idsWithErrors); + } + + scheduleJobsForRecordList(records); + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Set getRecordIdsWithErrors(List records) + { + return records.stream() + .filter(r -> !recordHasErrors().test(r)) + .map(r -> r.getValueInteger("id")) + .collect(Collectors.toSet()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postDelete(DeleteInput deleteInput, List records) throws QException + { + Set idsWithErrors = getRecordIdsWithErrors(records); + unscheduleJobsForRecordList(records, idsWithErrors); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void scheduleJobsForRecordList(List records) + { + List recordsWithoutErrors = records.stream().filter(recordHasErrors()).toList(); + if(CollectionUtils.nullSafeIsEmpty(recordsWithoutErrors)) + { + return; + } + + try + { + Map originalRecordMap = recordsWithoutErrors.stream().collect(Collectors.toMap(r -> r.getValueInteger("id"), r -> r)); + List freshRecordListWithAssociations = freshlyQueryForRecordsWithAssociations(recordsWithoutErrors); + + QScheduleManager scheduleManager = QScheduleManager.getInstance(); + for(QRecord record : freshRecordListWithAssociations) + { + try + { + scheduleManager.setupScheduledJob(new ScheduledJob(record)); + } + catch(Exception e) + { + LOG.warn("Caught exception while scheduling a job in post-action", e, logPair("id", record.getValue("id"))); + if(originalRecordMap.containsKey(record.getValueInteger("id"))) + { + originalRecordMap.get(record.getValueInteger("id")).addWarning(new QWarningMessage("Error scheduling job: " + e.getMessage())); + } + } + } + } + catch(Exception e) + { + LOG.warn("Error scheduling jobs in post-action", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Predicate recordHasErrors() + { + return r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static List freshlyQueryForRecordsWithAssociations(List records) throws QException + { + List idList = records.stream().map(r -> r.getValueInteger("id")).toList(); + + return new QueryAction().execute(new QueryInput(ScheduledJob.TABLE_NAME) + .withIncludeAssociations(true) + .withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, idList)))) + .getRecords(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void unscheduleJobsForRecordList(List oldRecords, Set exceptIdsWithErrors) + { + try + { + QScheduleManager scheduleManager = QScheduleManager.getInstance(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for un-schedule - use the old records as they are - don't re-query them (they may not exist anymore!) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : oldRecords) + { + try + { + ScheduledJob scheduledJob = new ScheduledJob(record); + + if(exceptIdsWithErrors.contains(scheduledJob.getId())) + { + LOG.info("Will not unschedule the job for a record that had an error", logPair("id", scheduledJob.getId())); + continue; + } + + scheduleManager.unscheduleScheduledJob(scheduledJob); + } + catch(Exception e) + { + LOG.warn("Caught exception while un-scheduling a job in post-action", e, logPair("id", record.getValue("id"))); + } + } + } + catch(Exception e) + { + LOG.warn("Error scheduling jobs in post-action", e); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java index 890d7c7f..7909963a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java @@ -372,7 +372,7 @@ public class MemoryRecordStore ///////////////////////////////////////////////// // set the next serial in the record if needed // ///////////////////////////////////////////////// - if(recordToInsert.getValue(primaryKeyField.getName()) == null && primaryKeyField.getType().equals(QFieldType.INTEGER)) + if(recordToInsert.getValue(primaryKeyField.getName()) == null && (primaryKeyField.getType().equals(QFieldType.INTEGER) || primaryKeyField.getType().equals(QFieldType.LONG))) { recordToInsert.setValue(primaryKeyField.getName(), nextSerial++); } @@ -384,6 +384,13 @@ public class MemoryRecordStore { nextSerial = recordToInsert.getValueInteger(primaryKeyField.getName()) + 1; } + else if(primaryKeyField.getType().equals(QFieldType.LONG) && recordToInsert.getValueLong(primaryKeyField.getName()) > nextSerial) + { + ////////////////////////////////////// + // todo - mmm, could overflow here? // + ////////////////////////////////////// + nextSerial = recordToInsert.getValueInteger(primaryKeyField.getName()) + 1; + } tableData.put(recordToInsert.getValue(primaryKeyField.getName()), recordToInsert); if(returnInsertedRecords) @@ -773,7 +780,7 @@ public class MemoryRecordStore { // todo - joins probably? QFieldMetaData field = table.getField(fieldName); - if(field.getType().equals(QFieldType.INTEGER) && (operator.equals(AggregateOperator.AVG))) + if((field.getType().equals(QFieldType.INTEGER) || field.getType().equals(QFieldType.LONG)) && (operator.equals(AggregateOperator.AVG))) { fieldType = QFieldType.DECIMAL; } @@ -809,6 +816,10 @@ public class MemoryRecordStore .filter(r -> r.getValue(fieldName) != null) .mapToInt(r -> r.getValueInteger(fieldName)) .sum(); + case LONG -> records.stream() + .filter(r -> r.getValue(fieldName) != null) + .mapToLong(r -> r.getValueLong(fieldName)) + .sum(); case DECIMAL -> records.stream() .filter(r -> r.getValue(fieldName) != null) .map(r -> r.getValueBigDecimal(fieldName)) @@ -823,6 +834,11 @@ public class MemoryRecordStore .mapToInt(r -> r.getValueInteger(fieldName)) .min() .stream().boxed().findFirst().orElse(null); + case LONG -> records.stream() + .filter(r -> r.getValue(fieldName) != null) + .mapToLong(r -> r.getValueLong(fieldName)) + .min() + .stream().boxed().findFirst().orElse(null); case DECIMAL, STRING, DATE, DATE_TIME -> { Optional serializable = records.stream() @@ -839,7 +855,12 @@ public class MemoryRecordStore { case INTEGER -> records.stream() .filter(r -> r.getValue(fieldName) != null) - .mapToInt(r -> r.getValueInteger(fieldName)) + .mapToLong(r -> r.getValueInteger(fieldName)) + .max() + .stream().boxed().findFirst().orElse(null); + case LONG -> records.stream() + .filter(r -> r.getValue(fieldName) != null) + .mapToLong(r -> r.getValueLong(fieldName)) .max() .stream().boxed().findFirst().orElse(null); case DECIMAL, STRING, DATE, DATE_TIME -> @@ -861,6 +882,11 @@ public class MemoryRecordStore .mapToInt(r -> r.getValueInteger(fieldName)) .average() .stream().boxed().findFirst().orElse(null); + case LONG -> records.stream() + .filter(r -> r.getValue(fieldName) != null) + .mapToLong(r -> r.getValueLong(fieldName)) + .average() + .stream().boxed().findFirst().orElse(null); case DECIMAL -> records.stream() .filter(r -> r.getValue(fieldName) != null) .mapToDouble(r -> r.getValueBigDecimal(fieldName).doubleValue()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java index 9fc87831..970b69ea 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java @@ -103,6 +103,7 @@ public class MockQueryAction implements QueryInterface { case STRING -> UUID.randomUUID().toString(); case INTEGER -> 42; + case LONG -> 42L; case DECIMAL -> new BigDecimal("3.14159"); case DATE -> LocalDate.of(1970, Month.JANUARY, 1); case DATE_TIME -> LocalDateTime.of(1970, Month.JANUARY, 1, 0, 0); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java index 84b3ab31..310fafe6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java @@ -55,6 +55,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; @@ -129,6 +130,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, QProcessMetaData processMetaData = new QProcessMetaData() .withName(NAME) + .withIcon(new QIcon().withName("healing")) .withStepList(List.of( new QFrontendStepMetaData() .withName("input") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java index 1c02a563..739619dd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProvi import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; @@ -71,6 +72,7 @@ public class RunTableAutomationsProcessStep implements BackendStep, MetaDataProd { QProcessMetaData processMetaData = new QProcessMetaData() .withName(NAME) + .withIcon(new QIcon().withName("directions_run")) .withStepList(List.of( new QFrontendStepMetaData() .withName("input") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 7f33a624..ae80f069 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -34,6 +34,7 @@ import java.util.Set; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; @@ -137,15 +138,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep // we do this, in case it needs to, for example, adjust values that // // are part of a unique key // ////////////////////////////////////////////////////////////////////// - Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole()); + Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); if(preInsertCustomizer.isPresent()) { - preInsertCustomizer.get().setInsertInput(insertInput); - preInsertCustomizer.get().setIsPreview(true); - AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().getWhenToRun(); + AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true); if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun)) { - List recordsAfterCustomizer = preInsertCustomizer.get().apply(runBackendStepInput.getRecords()); + List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, runBackendStepInput.getRecords(), true); runBackendStepInput.setRecords(recordsAfterCustomizer); /////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java index 172ec4cf..51dd30fe 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java @@ -104,12 +104,12 @@ public class BaseStreamedETLStep *******************************************************************************/ protected void moveReviewStepAfterValidateStep(RunBackendStepOutput runBackendStepOutput) { - LOG.info("Skipping to validation step"); + LOG.debug("Skipping to validation step"); ArrayList stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); - LOG.debug("Step list pre: " + stepList); + LOG.trace("Step list pre: " + stepList); stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)); stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); runBackendStepOutput.getProcessState().setStepList(stepList); - LOG.debug("Step list post: " + stepList); + LOG.trace("Step list post: " + stepList); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java index 3acfe8d4..4c3d75cd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java @@ -136,7 +136,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe asyncRecordPipeLoop.setMinRecordsToConsume(overrideRecordPipeCapacity); } - int recordCount = asyncRecordPipeLoop.run("StreamedETL>Execute>ExtractStep", null, recordPipe, (status) -> + int recordCount = asyncRecordPipeLoop.run("StreamedETLExecute>Extract>" + runBackendStepInput.getProcessName(), null, recordPipe, (status) -> { extractStep.run(runBackendStepInput, runBackendStepOutput); return (runBackendStepOutput); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java index 4921c9ce..e24e617a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -125,7 +125,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe // } List previewRecordList = new ArrayList<>(); - new AsyncRecordPipeLoop().run("StreamedETL>Preview>ExtractStep", PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> + new AsyncRecordPipeLoop().run("StreamedETLPreview>Extract>" + runBackendStepInput.getProcessName(), PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> { runBackendStepInput.setAsyncJobCallback(status); extractStep.run(runBackendStepInput, runBackendStepOutput); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java index ed177c52..12f584e2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java @@ -119,7 +119,7 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back transformStep.preRun(runBackendStepInput, runBackendStepOutput); List previewRecordList = new ArrayList<>(); - int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Preview>ValidateStep", null, recordPipe, (status) -> + int recordCount = new AsyncRecordPipeLoop().run("StreamedETLValidate>Extract>" + runBackendStepInput.getProcessName(), null, recordPipe, (status) -> { extractStep.run(runBackendStepInput, runBackendStepOutput); return (runBackendStepOutput); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 96a02287..8d3e8287 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.VariantRunStrategy; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; @@ -490,5 +491,28 @@ public class StreamedETLWithFrontendProcess return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Builder withVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + processMetaData.setVariantRunStrategy(variantRunStrategy); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Builder withVariantBackend(String variantBackend) + { + processMetaData.setVariantBackend(variantBackend); + return (this); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java index f17b31e0..5a2543b9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.VariantRunStrategy; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.ExtractViaBasepullQueryStep; @@ -248,5 +249,28 @@ public class TableSyncProcess super.withExtractStepClass(extractStepClass); return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Builder withVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + processMetaData.setVariantRunStrategy(variantRunStrategy); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Builder withVariantBackend(String variantBackend) + { + processMetaData.setVariantBackend(variantBackend); + return (this); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java new file mode 100644 index 00000000..a51999ab --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -0,0 +1,508 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.CapturedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; +import com.kingsrook.qqq.backend.core.logging.LogPair; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.VariantRunStrategy; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentityFactory; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableProcessRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableSQSQueueRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableTableAutomationsRunner; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** QQQ service to manage scheduled jobs, using 1 or more Schedulers - implementations + ** of the QSchedulerInterface + *******************************************************************************/ +public class QScheduleManager +{ + private static final QLogger LOG = QLogger.getLogger(QScheduleManager.class); + + private static QScheduleManager qScheduleManager = null; + private final QInstance qInstance; + private final Supplier systemUserSessionSupplier; + + private Map schedulers = new HashMap<>(); + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private QScheduleManager(QInstance qInstance, Supplier systemUserSessionSupplier) + { + this.qInstance = qInstance; + this.systemUserSessionSupplier = systemUserSessionSupplier; + } + + + + /******************************************************************************* + ** Singleton initiator - e.g., must be called to initially initialize the singleton + ** before anyone else calls getInstance (they'll get an error if they call that first). + *******************************************************************************/ + public static QScheduleManager initInstance(QInstance qInstance, Supplier systemUserSessionSupplier) throws QException + { + if(qScheduleManager == null) + { + qScheduleManager = new QScheduleManager(qInstance, systemUserSessionSupplier); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // initialize the scheduler(s) we're configured to use // + // do this, even if we won't start them - so, for example, a web server can still be aware of schedules in the application // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QSchedulerMetaData schedulerMetaData : CollectionUtils.nonNullMap(qInstance.getSchedulers()).values()) + { + QSchedulerInterface scheduler = schedulerMetaData.initSchedulerInstance(qInstance, systemUserSessionSupplier); + qScheduleManager.schedulers.put(schedulerMetaData.getName(), scheduler); + } + } + return (qScheduleManager); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void defineDefaultSchedulableTypesInInstance(QInstance qInstance) + { + qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.PROCESS.name()).withRunner(new QCodeReference(SchedulableProcessRunner.class))); + qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.QUEUE_PROCESSOR.name()).withRunner(new QCodeReference(SchedulableSQSQueueRunner.class))); + qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.TABLE_AUTOMATIONS.name()).withRunner(new QCodeReference(SchedulableTableAutomationsRunner.class))); + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static QScheduleManager getInstance() + { + if(qScheduleManager == null) + { + throw (new IllegalStateException("QScheduleManager singleton has not been init'ed (call initInstance).")); + } + return (qScheduleManager); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void start() throws QException + { + ////////////////////////////////////////////////////////////////////////// + // exit w/o starting schedulers, if schedule manager isn't enabled here // + ////////////////////////////////////////////////////////////////////////// + if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) + { + LOG.info("Not starting ScheduleManager per settings."); + schedulers.values().forEach(s -> s.doNotStart()); + return; + } + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // ensure that everything which should be scheduled is scheduled, in the appropriate scheduler // + ///////////////////////////////////////////////////////////////////////////////////////////////// + QContext.withTemporaryContext(new CapturedContext(qInstance, systemUserSessionSupplier.get()), () -> setupAllSchedules()); + + ////////////////////////// + // start each scheduler // + ////////////////////////// + schedulers.values().forEach(s -> s.start()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void stop() + { + schedulers.values().forEach(s -> s.stop()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void stopAsync() + { + schedulers.values().forEach(s -> s.stopAsync()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void setupAllSchedules() throws QException + { + ///////////////////////////////////////////// + // read dynamic schedules // + // e.g., user-scheduled processes, reports // + ///////////////////////////////////////////// + List scheduledJobList = null; + try + { + if(QContext.getQInstance().getTables().containsKey(ScheduledJob.TABLE_NAME)) + { + scheduledJobList = new QueryAction() + .execute(new QueryInput(ScheduledJob.TABLE_NAME) + .withIncludeAssociations(true)) + .getRecordEntities(ScheduledJob.class); + } + } + catch(Exception e) + { + throw (new QRuntimeException("Failed to query for scheduled jobs - will not set up scheduler!", e)); + } + + ///////////////////////////////////////////////////////// + // let the schedulers know we're starting this process // + ///////////////////////////////////////////////////////// + schedulers.values().forEach(s -> s.startOfSetupSchedules()); + + ///////////////////////// + // schedule all queues // + ///////////////////////// + for(QQueueMetaData queue : qInstance.getQueues().values()) + { + if(queue.getSchedule() != null) + { + setupQueue(queue); + } + } + + //////////////////////////////////////// + // schedule all tables w/ automations // + //////////////////////////////////////// + for(QTableMetaData table : qInstance.getTables().values()) + { + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + if(automationDetails != null && automationDetails.getSchedule() != null) + { + setupTableAutomations(table); + } + } + + ///////////////////////////////////////// + // schedule all processes that need it // + ///////////////////////////////////////// + for(QProcessMetaData process : qInstance.getProcesses().values()) + { + if(process.getSchedule() != null) + { + setupProcess(process); + } + } + + ///////////////////////////////////////////////////////////////////////////////////////////// + // todo- before, or after meta-datas? // + // like quartz, it'd just re-schedule if a dupe - but, should we do our own dupe checking? // + ///////////////////////////////////////////////////////////////////////////////////////////// + for(ScheduledJob scheduledJob : CollectionUtils.nonNullList(scheduledJobList)) + { + try + { + setupScheduledJob(scheduledJob); + } + catch(Exception e) + { + LOG.info("Caught exception while scheduling a job", e, logPair("id", scheduledJob.getId())); + } + } + + ////////////////////////////////////////////////////////// + // let the schedulers know we're done with this process // + ////////////////////////////////////////////////////////// + schedulers.values().forEach(s -> s.endOfSetupSchedules()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void setupScheduledJob(ScheduledJob scheduledJob) throws QException + { + BasicSchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(scheduledJob); + + //////////////////////////////////////////////////////////////////////////////// + // non-active jobs should be deleted from the scheduler. they get re-added // + // if they get re-activated. but we don't want to rely on (e.g., for quartz) // + // the paused state to be drive by is-active. else, devops-pause & unpause // + // operations would clobber scheduled-job record facts // + //////////////////////////////////////////////////////////////////////////////// + if(!scheduledJob.getIsActive()) + { + unscheduleScheduledJob(scheduledJob); + return; + } + + String exceptionSuffix = "in scheduledJob [" + scheduledJob.getId() + "]"; + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // setup schedule meta-data object based on schedule data in the scheduled job - throwing if not well populated // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(scheduledJob.getRepeatSeconds() == null && !StringUtils.hasContent(scheduledJob.getCronExpression())) + { + throw (new QException("Missing a schedule (cronString or repeatSeconds) " + exceptionSuffix)); + } + + QScheduleMetaData scheduleMetaData = new QScheduleMetaData(); + scheduleMetaData.setCronExpression(scheduledJob.getCronExpression()); + scheduleMetaData.setCronTimeZoneId(scheduledJob.getCronTimeZoneId()); + scheduleMetaData.setRepeatSeconds(scheduledJob.getRepeatSeconds()); + + ///////////////////////////////// + // get & validate the job type // + ///////////////////////////////// + if(!StringUtils.hasContent(scheduledJob.getType())) + { + throw (new QException("Missing a type " + exceptionSuffix)); + } + + SchedulableType schedulableType = qInstance.getSchedulableType(scheduledJob.getType()); + if(schedulableType == null) + { + throw (new QException("Unrecognized type [" + scheduledJob.getType() + "] " + exceptionSuffix)); + } + + QSchedulerInterface scheduler = getScheduler(scheduledJob.getSchedulerName()); + Map paramMap = new HashMap<>(scheduledJob.getJobParametersMap()); + + SchedulableRunner runner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + runner.validateParams(schedulableIdentity, new HashMap<>(paramMap)); + + scheduler.setupSchedulable(schedulableIdentity, schedulableType, paramMap, scheduleMetaData, true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void unscheduleAll() + { + schedulers.values().forEach(s -> + { + try + { + s.unscheduleAll(); + } + catch(Exception e) + { + LOG.warn("Error unscheduling everything in scheduler " + s, e); + } + }); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void unscheduleScheduledJob(ScheduledJob scheduledJob) throws QException + { + QSchedulerInterface scheduler = getScheduler(scheduledJob.getSchedulerName()); + + BasicSchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(scheduledJob); + SchedulableType schedulableType = qInstance.getSchedulableType(scheduledJob.getType()); + + scheduler.unscheduleSchedulable(schedulableIdentity, schedulableType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupProcess(QProcessMetaData process) throws QException + { + BasicSchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(process); + QSchedulerInterface scheduler = getScheduler(process.getSchedule().getSchedulerName()); + boolean allowedToStart = SchedulerUtils.allowedToStart(process.getName()); + + Map paramMap = new HashMap<>(); + paramMap.put("processName", process.getName()); + + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.PROCESS.name()); + + if(process.getVariantBackend() == null || VariantRunStrategy.SERIAL.equals(process.getVariantRunStrategy())) + { + /////////////////////////////////////////////// + // if no variants, or variant is serial mode // + /////////////////////////////////////////////// + scheduler.setupSchedulable(schedulableIdentity, schedulableType, new HashMap<>(paramMap), process.getSchedule(), allowedToStart); + } + else if(VariantRunStrategy.PARALLEL.equals(process.getVariantRunStrategy())) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // if this a "parallel", which for example means we want to have a thread for each backend variant // + // running at the same time, get the variant records and schedule each separately // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + QBackendMetaData backendMetaData = qInstance.getBackend(process.getVariantBackend()); + for(QRecord qRecord : CollectionUtils.nonNullList(SchedulerUtils.getBackendVariantFilteredRecords(process))) + { + try + { + HashMap parameters = new HashMap<>(paramMap); + HashMap variantMap = new HashMap<>(Map.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); + parameters.put("backendVariantData", variantMap); + + String identity = schedulableIdentity.getIdentity() + ";" + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()); + String description = schedulableIdentity.getDescription() + " for variant: " + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()); + + BasicSchedulableIdentity variantIdentity = new BasicSchedulableIdentity(identity, description); + + scheduler.setupSchedulable(variantIdentity, schedulableType, parameters, process.getSchedule(), allowedToStart); + } + catch(Exception e) + { + LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); + } + } + } + else + { + LOG.error("Unsupported Schedule Run Strategy [" + process.getVariantRunStrategy() + "] was provided."); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupTableAutomations(QTableMetaData table) throws QException + { + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.TABLE_AUTOMATIONS.name()); + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + QSchedulerInterface scheduler = getScheduler(automationDetails.getSchedule().getSchedulerName()); + + List tableActionList = PollingAutomationPerTableRunner.getTableActions(qInstance, automationDetails.getProviderName()) + .stream().filter(ta -> ta.tableName().equals(table.getName())) + .toList(); + + for(PollingAutomationPerTableRunner.TableActionsInterface tableActions : tableActionList) + { + SchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(tableActions); + boolean allowedToStart = SchedulerUtils.allowedToStart(table.getName()); + + Map paramMap = new HashMap<>(); + paramMap.put("tableName", tableActions.tableName()); + paramMap.put("automationStatus", tableActions.status().name()); + scheduler.setupSchedulable(schedulableIdentity, schedulableType, new HashMap<>(paramMap), automationDetails.getSchedule(), allowedToStart); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupQueue(QQueueMetaData queue) throws QException + { + SchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(queue); + QSchedulerInterface scheduler = getScheduler(queue.getSchedule().getSchedulerName()); + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.QUEUE_PROCESSOR.name()); + boolean allowedToStart = SchedulerUtils.allowedToStart(queue.getName()); + + Map paramMap = new HashMap<>(); + paramMap.put("queueName", queue.getName()); + scheduler.setupSchedulable(schedulableIdentity, schedulableType, new HashMap<>(paramMap), queue.getSchedule(), allowedToStart); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QSchedulerInterface getScheduler(String schedulerName) throws QException + { + if(!StringUtils.hasContent(schedulerName)) + { + throw (new QException("Scheduler name was not given (and the concept of a default scheduler does not exist at this time).")); + } + + QSchedulerInterface scheduler = schedulers.get(schedulerName); + if(scheduler == null) + { + throw (new QException("Unrecognized schedulerName [" + schedulerName + "]")); + } + + return (scheduler); + } + + + + /******************************************************************************* + ** reset the singleton instance (to null); clear the map of schedulers. + ** Not clear it's ever useful to call in main-code - but can be used for tests. + *******************************************************************************/ + public void unInit() + { + qScheduleManager = null; + schedulers.values().forEach(s -> s.unInit()); + schedulers.clear(); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java new file mode 100644 index 00000000..b9f16902 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java @@ -0,0 +1,113 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.io.Serializable; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface QSchedulerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + String getSchedulerName(); + + /******************************************************************************* + ** + *******************************************************************************/ + void start(); + + /******************************************************************************* + ** called to indicate that the schedule manager is past its startup routine, + ** but that the schedule should not actually be running in this process. + *******************************************************************************/ + default void doNotStart() + { + + } + + /******************************************************************************* + ** + *******************************************************************************/ + void setupSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType, Map parameters, QScheduleMetaData schedule, boolean allowedToStart); + + /******************************************************************************* + ** + *******************************************************************************/ + void unscheduleSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType); + + /******************************************************************************* + ** + *******************************************************************************/ + void unscheduleAll() throws QException; + + /******************************************************************************* + ** + *******************************************************************************/ + void stopAsync(); + + /******************************************************************************* + ** + *******************************************************************************/ + void stop(); + + /******************************************************************************* + ** Handle a whole shutdown of the scheduler system (e.g., between unit tests). + *******************************************************************************/ + default void unInit() + { + ///////////////////// + // noop by default // + ///////////////////// + } + + /******************************************************************************* + ** let the scheduler know when the schedule manager is at the start of setting up schedules. + *******************************************************************************/ + default void startOfSetupSchedules() + { + ///////////////////// + // noop by default // + ///////////////////// + } + + /******************************************************************************* + ** let the scheduler know when the schedule manager is at the end of setting up schedules. + *******************************************************************************/ + default void endOfSetupSchedules() + { + ///////////////////// + // noop by default // + ///////////////////// + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java deleted file mode 100644 index 375aebb9..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java +++ /dev/null @@ -1,502 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.scheduler; - - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.function.Supplier; -import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; -import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; -import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; -import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; -import com.kingsrook.qqq.backend.core.logging.LogPair; -import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; -import com.kingsrook.qqq.backend.core.model.session.QSession; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; - - -/******************************************************************************* - ** QQQ Service (Singleton) that starts up repeating, scheduled jobs within QQQ. - ** - ** These include: - ** - Automation providers (which require polling) - ** - Queue pollers - ** - Scheduled processes. - ** - ** All of these jobs run using a "system session" - as defined by the sessionSupplier. - *******************************************************************************/ -public class ScheduleManager -{ - private static final QLogger LOG = QLogger.getLogger(ScheduleManager.class); - - private static ScheduleManager scheduleManager = null; - private final QInstance qInstance; - - protected Supplier sessionSupplier; - - ///////////////////////////////////////////////////////////////////////////////////// - // for jobs that don't define a delay index, auto-stagger them, using this counter // - ///////////////////////////////////////////////////////////////////////////////////// - private int delayIndex = 0; - - private List executors = new ArrayList<>(); - - - - /******************************************************************************* - ** Singleton constructor - *******************************************************************************/ - private ScheduleManager(QInstance qInstance) - { - this.qInstance = qInstance; - } - - - - /******************************************************************************* - ** Singleton accessor - *******************************************************************************/ - public static ScheduleManager getInstance(QInstance qInstance) - { - if(scheduleManager == null) - { - scheduleManager = new ScheduleManager(qInstance); - } - return (scheduleManager); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public void start() - { - if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) - { - LOG.info("Not starting ScheduleManager per settings."); - return; - } - - boolean needToClearContext = false; - try - { - if(QContext.getQInstance() == null) - { - needToClearContext = true; - QContext.init(qInstance, sessionSupplier.get()); - } - - for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values()) - { - startQueueProvider(queueProvider); - } - - for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values()) - { - startAutomationProviderPerTable(automationProvider); - } - - for(QProcessMetaData process : qInstance.getProcesses().values()) - { - if(process.getSchedule() != null && allowedToStart(process.getName())) - { - QScheduleMetaData scheduleMetaData = process.getSchedule(); - if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) - { - /////////////////////////////////////////////// - // if no variants, or variant is serial mode // - /////////////////////////////////////////////// - startProcess(process, null); - } - else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) - { - ///////////////////////////////////////////////////////////////////////////////////////////////////// - // if this a "parallel", which for example means we want to have a thread for each backend variant // - // running at the same time, get the variant records and schedule each separately // - ///////////////////////////////////////////////////////////////////////////////////////////////////// - QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); - for(QRecord qRecord : CollectionUtils.nonNullList(getBackendVariantFilteredRecords(process))) - { - try - { - startProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); - } - catch(Exception e) - { - LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); - } - } - } - else - { - LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided."); - } - } - } - } - finally - { - if(needToClearContext) - { - QContext.clear(); - } - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private List getBackendVariantFilteredRecords(QProcessMetaData processMetaData) - { - List records = null; - try - { - QScheduleMetaData scheduleMetaData = processMetaData.getSchedule(); - QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); - - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(backendMetaData.getVariantOptionsTableName()); - queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backendMetaData.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backendMetaData.getVariantOptionsTableTypeValue()))); - - QContext.init(qInstance, sessionSupplier.get()); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - records = queryOutput.getRecords(); - } - catch(Exception e) - { - LOG.error("An error fetching variant data for process [" + processMetaData.getLabel() + "]", e); - } - - return (records); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void startAutomationProviderPerTable(QAutomationProviderMetaData automationProvider) - { - /////////////////////////////////////////////////////////////////////////////////// - // ask the PollingAutomationPerTableRunner how many threads of itself need setup // - // then start a scheduled executor foreach one // - /////////////////////////////////////////////////////////////////////////////////// - List tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName()); - for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions) - { - if(allowedToStart(tableAction.tableName())) - { - PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProvider.getName(), sessionSupplier, tableAction); - StandardScheduledExecutor executor = new StandardScheduledExecutor(runner); - - QScheduleMetaData schedule = Objects.requireNonNullElseGet(automationProvider.getSchedule(), this::getDefaultSchedule); - - executor.setName(runner.getName()); - setScheduleInExecutor(schedule, executor); - if(!executor.start()) - { - LOG.warn("executor.start return false for: " + executor.getName()); - } - - executors.add(executor); - } - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private boolean allowedToStart(String name) - { - String propertyName = "qqq.scheduleManager.onlyStartNamesMatching"; - String propertyValue = System.getProperty(propertyName, ""); - if(propertyValue.equals("")) - { - return (true); - } - - return (name.matches(propertyValue)); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void startQueueProvider(QQueueProviderMetaData queueProvider) - { - if(allowedToStart(queueProvider.getName())) - { - switch(queueProvider.getType()) - { - case SQS: - startSqsProvider((SQSQueueProviderMetaData) queueProvider); - break; - default: - throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType()); - } - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void startSqsProvider(SQSQueueProviderMetaData queueProvider) - { - QInstance scheduleManagerQueueInstance = qInstance; - Supplier scheduleManagerSessionSupplier = sessionSupplier; - - for(QQueueMetaData queue : qInstance.getQueues().values()) - { - if(queueProvider.getName().equals(queue.getProviderName()) && allowedToStart(queue.getName())) - { - SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); - sqsQueuePoller.setQueueProviderMetaData(queueProvider); - sqsQueuePoller.setQueueMetaData(queue); - sqsQueuePoller.setQInstance(scheduleManagerQueueInstance); - sqsQueuePoller.setSessionSupplier(scheduleManagerSessionSupplier); - - StandardScheduledExecutor executor = new StandardScheduledExecutor(sqsQueuePoller); - - QScheduleMetaData schedule = Objects.requireNonNullElseGet(queue.getSchedule(), - () -> Objects.requireNonNullElseGet(queueProvider.getSchedule(), - this::getDefaultSchedule)); - - executor.setName(queue.getName()); - setScheduleInExecutor(schedule, executor); - if(!executor.start()) - { - LOG.warn("executor.start return false for: " + executor.getName()); - } - - executors.add(executor); - } - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void startProcess(QProcessMetaData process, Map backendVariantData) - { - Runnable runProcess = () -> - { - String originalThreadName = Thread.currentThread().getName(); - - try - { - if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) - { - QContext.init(qInstance, sessionSupplier.get()); - executeSingleProcess(process, backendVariantData); - } - else if(QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) - { - /////////////////////////////////////////////////////////////////////////////////////////////////// - // if this is "serial", which for example means we want to run each backend variant one after // - // the other in the same thread so loop over these here so that they run in same lambda function // - /////////////////////////////////////////////////////////////////////////////////////////////////// - for(QRecord qRecord : getBackendVariantFilteredRecords(process)) - { - try - { - QContext.init(qInstance, sessionSupplier.get()); - QScheduleMetaData scheduleMetaData = process.getSchedule(); - QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); - executeSingleProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); - } - catch(Exception e) - { - LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); - } - } - } - } - catch(Exception e) - { - LOG.warn("Exception thrown running scheduled process [" + process.getName() + "]", e); - } - finally - { - Thread.currentThread().setName(originalThreadName); - QContext.clear(); - } - }; - - StandardScheduledExecutor executor = new StandardScheduledExecutor(runProcess); - executor.setName("process:" + process.getName()); - setScheduleInExecutor(process.getSchedule(), executor); - if(!executor.start()) - { - LOG.warn("executor.start return false for: " + executor.getName()); - } - - executors.add(executor); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private static void executeSingleProcess(QProcessMetaData process, Map backendVariantData) throws QException - { - if(backendVariantData != null) - { - QContext.getQSession().setBackendVariants(backendVariantData); - } - - Thread.currentThread().setName("ScheduledProcess>" + process.getName()); - LOG.debug("Running Scheduled Process [" + process.getName() + "]"); - - RunProcessInput runProcessInput = new RunProcessInput(); - runProcessInput.setProcessName(process.getName()); - runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); - - QContext.pushAction(runProcessInput); - - RunProcessAction runProcessAction = new RunProcessAction(); - runProcessAction.execute(runProcessInput); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void setScheduleInExecutor(QScheduleMetaData schedule, StandardScheduledExecutor executor) - { - if(schedule.getRepeatMillis() != null) - { - executor.setDelayMillis(schedule.getRepeatMillis()); - } - else - { - executor.setDelayMillis(1000 * schedule.getRepeatSeconds()); - } - - if(schedule.getInitialDelayMillis() != null) - { - executor.setInitialDelayMillis(schedule.getInitialDelayMillis()); - } - else if(schedule.getInitialDelaySeconds() != null) - { - executor.setInitialDelayMillis(1000 * schedule.getInitialDelaySeconds()); - } - else - { - executor.setInitialDelayMillis(1000 * ++delayIndex); - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private QScheduleMetaData getDefaultSchedule() - { - QScheduleMetaData schedule; - schedule = new QScheduleMetaData() - .withInitialDelaySeconds(delayIndex++) - .withRepeatSeconds(60); - return schedule; - } - - - - /******************************************************************************* - ** Setter for sessionSupplier - ** - *******************************************************************************/ - public void setSessionSupplier(Supplier sessionSupplier) - { - this.sessionSupplier = sessionSupplier; - } - - - - /******************************************************************************* - ** Getter for managedExecutors - ** - *******************************************************************************/ - public List getExecutors() - { - return executors; - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public void stopAsync() - { - for(StandardScheduledExecutor scheduledExecutor : executors) - { - scheduledExecutor.stopAsync(); - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - static void resetSingleton() - { - scheduleManager = null; - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java new file mode 100644 index 00000000..6feeb7ee --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java @@ -0,0 +1,181 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.LogPair; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.VariantRunStrategy; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; + + +/******************************************************************************* + ** Utility methods used by various schedulers. + *******************************************************************************/ +public class SchedulerUtils +{ + private static final QLogger LOG = QLogger.getLogger(SchedulerUtils.class); + + + /******************************************************************************* + ** + *******************************************************************************/ + public static boolean allowedToStart(String name) + { + String propertyName = "qqq.scheduleManager.onlyStartNamesMatching"; + String propertyValue = System.getProperty(propertyName, ""); + if(propertyValue.equals("")) + { + return (true); + } + + return (name.matches(propertyValue)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void runProcess(QInstance qInstance, Supplier sessionSupplier, QProcessMetaData process, Map backendVariantData, Map processInputValues) + { + String originalThreadName = Thread.currentThread().getName(); + + try + { + QContext.init(qInstance, sessionSupplier.get()); + + if(process.getVariantBackend() == null || VariantRunStrategy.PARALLEL.equals(process.getVariantRunStrategy())) + { + executeSingleProcess(process, backendVariantData, processInputValues); + } + else if(VariantRunStrategy.SERIAL.equals(process.getVariantRunStrategy())) + { + /////////////////////////////////////////////////////////////////////////////////////////////////// + // if this is "serial", which for example means we want to run each backend variant one after // + // the other in the same thread so loop over these here so that they run in same lambda function // + /////////////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord qRecord : getBackendVariantFilteredRecords(process)) + { + try + { + QBackendMetaData backendMetaData = qInstance.getBackend(process.getVariantBackend()); + Map thisVariantData = MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())); + executeSingleProcess(process, thisVariantData, processInputValues); + } + catch(Exception e) + { + LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); + } + } + } + } + catch(Exception e) + { + LOG.warn("Exception thrown running scheduled process [" + process.getName() + "]", e); + } + finally + { + Thread.currentThread().setName(originalThreadName); + QContext.clear(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void executeSingleProcess(QProcessMetaData process, Map backendVariantData, Map processInputValues) throws QException + { + if(backendVariantData != null) + { + QContext.getQSession().setBackendVariants(backendVariantData); + } + + Thread.currentThread().setName("ScheduledProcess>" + process.getName()); + LOG.debug("Running Scheduled Process [" + process.getName() + "] with values [" + processInputValues + "]"); + + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(process.getName()); + + for(Map.Entry entry : CollectionUtils.nonNullMap(processInputValues).entrySet()) + { + runProcessInput.withValue(entry.getKey(), entry.getValue()); + } + + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + + QContext.pushAction(runProcessInput); + + RunProcessAction runProcessAction = new RunProcessAction(); + runProcessAction.execute(runProcessInput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static List getBackendVariantFilteredRecords(QProcessMetaData processMetaData) + { + List records = null; + try + { + QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(processMetaData.getVariantBackend()); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(backendMetaData.getVariantOptionsTableName()); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backendMetaData.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backendMetaData.getVariantOptionsTableTypeValue()))); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + records = queryOutput.getRecords(); + } + catch(Exception e) + { + LOG.error("An error fetching variant data for process [" + processMetaData.getLabel() + "]", e); + } + + return (records); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcess.java new file mode 100644 index 00000000..9c3840ca --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcess.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; + + +/******************************************************************************* + ** Management process to reschedule all scheduled jobs (in all schedulers). + *******************************************************************************/ +public class RescheduleAllJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Reschedule all Scheduled Jobs") + .withIcon(new QIcon("update")) + .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to reschedule all jobs."))), + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All jobs have been rescheduled."))))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QScheduleManager.getInstance().setupAllSchedules(); + } + catch(Exception e) + { + throw (new QException("Error setting up all scheduled jobs.", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcess.java new file mode 100644 index 00000000..11492b06 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcess.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; + + +/******************************************************************************* + ** Management process to unschedule all scheduled jobs (in all schedulers). + *******************************************************************************/ +public class UnscheduleAllJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Unschedule all Scheduled Jobs") + .withIcon(new QIcon("update_disabled")) + .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to unschedule all jobs."))), + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All jobs have been unscheduled."))))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QScheduleManager.getInstance().unscheduleAll(); + } + catch(Exception e) + { + throw (new QException("Error unscheduling all scheduled jobs.", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobAndTriggerWrapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobAndTriggerWrapper.java new file mode 100644 index 00000000..62ec6cb7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobAndTriggerWrapper.java @@ -0,0 +1,34 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import org.quartz.JobDetail; +import org.quartz.Trigger; + + +/******************************************************************************* + ** + *******************************************************************************/ +public record QuartzJobAndTriggerWrapper(JobDetail jobDetail, Trigger trigger, Trigger.TriggerState triggerState) +{ +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java new file mode 100644 index 00000000..f635bd9b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java @@ -0,0 +1,109 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.context.CapturedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; +import org.apache.logging.log4j.Level; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +@DisallowConcurrentExecution +public class QuartzJobRunner implements Job +{ + private static final QLogger LOG = QLogger.getLogger(QuartzJobRunner.class); + + private static Level logLevel = null; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void execute(JobExecutionContext context) throws JobExecutionException + { + CapturedContext capturedContext = QContext.capture(); + + String name = null; + SchedulableType schedulableType = null; + Map params = null; + try + { + name = context.getJobDetail().getKey().getName(); + + QuartzScheduler quartzScheduler = QuartzScheduler.getInstance(); + QInstance qInstance = quartzScheduler.getQInstance(); + QContext.init(qInstance, quartzScheduler.getSessionSupplier().get()); + + schedulableType = qInstance.getSchedulableType(context.getJobDetail().getJobDataMap().getString("type")); + params = (Map) context.getJobDetail().getJobDataMap().get("params"); + + SchedulableRunner schedulableRunner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + + if(logLevel != null) + { + LOG.log(logLevel, "Running QuartzJob", null, logPair("name", name), logPair("type", schedulableType.getName()), logPair("params", params)); + } + + schedulableRunner.run(params); + + if(logLevel != null) + { + LOG.log(logLevel, "Finished QuartzJob", null, logPair("name", name), logPair("type", schedulableType.getName()), logPair("params", params)); + } + } + catch(Exception e) + { + LOG.warn("Error running QuartzJob", e, logPair("name", name), logPair("type", schedulableType == null ? null : schedulableType.getName()), logPair("params", params)); + } + finally + { + QContext.init(capturedContext); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void setLogLevel(Level level) + { + logLevel = level; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java new file mode 100644 index 00000000..f41362c5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java @@ -0,0 +1,742 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import java.io.Serializable; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.TimeZone; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.AnyKey; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; +import org.quartz.CronExpression; +import org.quartz.CronScheduleBuilder; +import org.quartz.Job; +import org.quartz.JobBuilder; +import org.quartz.JobDetail; +import org.quartz.JobKey; +import org.quartz.ScheduleBuilder; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.SimpleScheduleBuilder; +import org.quartz.Trigger; +import org.quartz.TriggerBuilder; +import org.quartz.TriggerKey; +import org.quartz.impl.StdSchedulerFactory; +import org.quartz.impl.matchers.GroupMatcher; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Singleton to provide access between QQQ and the quartz Scheduler system. + *******************************************************************************/ +public class QuartzScheduler implements QSchedulerInterface +{ + private static final QLogger LOG = QLogger.getLogger(QuartzScheduler.class); + + private static QuartzScheduler quartzScheduler = null; + + private final QInstance qInstance; + private String schedulerName; + private Supplier sessionSupplier; + + private Scheduler scheduler; + + + ///////////////////////////////////////////////////////////////////////////////////////// + // create memoization objects for some quartz-query functions, that we'll only want to // + // use during our setup routine, when we'd query it many times over and over again. // + // So default to a timeout of 0 (effectively disabling memoization). then in the // + // start-of-setup and end-of-setup methods, temporarily increase, then re-decrease // + ///////////////////////////////////////////////////////////////////////////////////////// + private Memoization> jobGroupNamesMemoization = new Memoization>() + .withTimeout(Duration.of(0, ChronoUnit.SECONDS)); + + private Memoization> jobKeyNamesMemoization = new Memoization>() + .withTimeout(Duration.of(0, ChronoUnit.SECONDS)); + + private Memoization> queryQuartzMemoization = new Memoization>() + .withTimeout(Duration.of(0, ChronoUnit.SECONDS)); + + private List> allMemoizations = List.of(jobGroupNamesMemoization, jobKeyNamesMemoization, queryQuartzMemoization); + + /////////////////////////////////////////////////////////////////////////////// + // vars used during the setup routine, to figure out what jobs need deleted. // + /////////////////////////////////////////////////////////////////////////////// + private boolean insideSetup = false; + private List scheduledJobsAtStartOfSetup = new ArrayList<>(); + private List scheduledJobsAtEndOfSetup = new ArrayList<>(); + + ///////////////////////////////////////////////////////////////////////////////// + // track if the instance is past the server's startup routine. // + // for quartz - we'll use this to know if we're allowed to schedule jobs. // + // that is - during server startup, we don't want to the schedule & unschedule // + // routine, which could potentially have serve concurrency problems // + ///////////////////////////////////////////////////////////////////////////////// + private boolean pastStartup = false; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + private QuartzScheduler(QInstance qInstance, String schedulerName, Supplier sessionSupplier) + { + this.qInstance = qInstance; + this.schedulerName = schedulerName; + this.sessionSupplier = sessionSupplier; + } + + + + /******************************************************************************* + ** Singleton initiator - e.g., must be called to initially initialize the singleton + ** before anyone else calls getInstance (they'll get an error if they call that first). + *******************************************************************************/ + public static QuartzScheduler initInstance(QInstance qInstance, String schedulerName, Properties quartzProperties, Supplier sessionSupplier) throws SchedulerException + { + if(quartzScheduler == null) + { + quartzScheduler = new QuartzScheduler(qInstance, schedulerName, sessionSupplier); + + /////////////////////////////////////////////////////////// + // Grab the Scheduler instance from the Factory // + // initialize it with the properties we took in as input // + /////////////////////////////////////////////////////////// + StdSchedulerFactory schedulerFactory = new StdSchedulerFactory(); + schedulerFactory.initialize(quartzProperties); + quartzScheduler.scheduler = schedulerFactory.getScheduler(); + } + return (quartzScheduler); + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static QuartzScheduler getInstance() + { + if(quartzScheduler == null) + { + throw (new IllegalStateException("QuartzScheduler singleton has not been init'ed (call initInstance).")); + } + return (quartzScheduler); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getSchedulerName() + { + return (schedulerName); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void start() + { + this.pastStartup = true; + + try + { + ////////////////////// + // and start it off // + ////////////////////// + scheduler.start(); + } + catch(Exception e) + { + LOG.error("Error starting quartz scheduler", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void doNotStart() + { + this.pastStartup = true; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stop() + { + try + { + scheduler.shutdown(true); + } + catch(SchedulerException e) + { + LOG.error("Error shutting down (stopping) quartz scheduler", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stopAsync() + { + try + { + scheduler.shutdown(false); + } + catch(SchedulerException e) + { + LOG.error("Error shutting down (stopping) quartz scheduler", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setupSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType, Map parameters, QScheduleMetaData schedule, boolean allowedToStart) + { + //////////////////////////////////////////////////////////////////////////// + // only actually schedule things if we're past the server startup routine // + //////////////////////////////////////////////////////////////////////////// + if(!pastStartup) + { + return; + } + + Map jobData = new HashMap<>(); + jobData.put("params", parameters); + jobData.put("type", schedulableType.getName()); + + scheduleJob(schedulableIdentity, schedulableType.getName(), QuartzJobRunner.class, jobData, schedule, allowedToStart); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void startOfSetupSchedules() + { + //////////////////////////////////////////////////////////////////////////// + // only actually schedule things if we're past the server startup routine // + //////////////////////////////////////////////////////////////////////////// + if(!pastStartup) + { + return; + } + + this.insideSetup = true; + this.allMemoizations.forEach(m -> m.setTimeout(Duration.ofSeconds(5))); + + try + { + this.scheduledJobsAtStartOfSetup = queryQuartz(); + this.scheduledJobsAtEndOfSetup = new ArrayList<>(); + } + catch(Exception e) + { + LOG.warn("Error querying quartz for the currently scheduled jobs during startup - will not be able to delete no-longer-needed jobs!", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void endOfSetupSchedules() + { + //////////////////////////////////////////////////////////////////////////// + // only actually schedule things if we're past the server startup routine // + //////////////////////////////////////////////////////////////////////////// + if(!pastStartup) + { + return; + } + + this.insideSetup = false; + this.allMemoizations.forEach(m -> m.setTimeout(Duration.ofSeconds(0))); + + if(this.scheduledJobsAtStartOfSetup == null) + { + return; + } + + try + { + Set startJobKeys = this.scheduledJobsAtStartOfSetup.stream().map(w -> w.jobDetail().getKey()).collect(Collectors.toSet()); + Set endJobKeys = this.scheduledJobsAtEndOfSetup.stream().map(w -> w.jobDetail().getKey()).collect(Collectors.toSet()); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // remove all 'end' keys from the set of start keys. any left-over start-keys need to be deleted. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + startJobKeys.removeAll(endJobKeys); + for(JobKey jobKey : startJobKeys) + { + LOG.info("Deleting job that had previously been scheduled, but doesn't appear to be any more", logPair("jobKey", jobKey)); + deleteJob(jobKey); + } + } + catch(Exception e) + { + LOG.warn("Error trying to clean up no-longer-needed jobs at end of scheduler setup", e); + } + + //////////////////////////////////////////////////// + // reset these lists, no need to keep them around // + //////////////////////////////////////////////////// + this.scheduledJobsAtStartOfSetup = null; + this.scheduledJobsAtEndOfSetup = null; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean scheduleJob(SchedulableIdentity schedulableIdentity, String groupName, Class jobClass, Map jobData, QScheduleMetaData scheduleMetaData, boolean allowedToStart) + { + try + { + ///////////////////////// + // Define job instance // + ///////////////////////// + JobKey jobKey = new JobKey(schedulableIdentity.getIdentity(), groupName); + JobDetail jobDetail = JobBuilder.newJob(jobClass) + .withIdentity(jobKey) + .withDescription(schedulableIdentity.getDescription()) + .storeDurably() + .requestRecovery() // todo - our frequent repeaters, maybe nice to say false here + .build(); + + jobDetail.getJobDataMap().putAll(jobData); + + ///////////////////////////////////////////////////////// + // map the qqq schedule meta data to a quartz schedule // + ///////////////////////////////////////////////////////// + ScheduleBuilder scheduleBuilder; + if(scheduleMetaData.isCron()) + { + CronExpression cronExpression = new CronExpression(scheduleMetaData.getCronExpression()); + cronExpression.setTimeZone(TimeZone.getTimeZone(scheduleMetaData.getCronTimeZoneId())); + scheduleBuilder = CronScheduleBuilder.cronSchedule(cronExpression); + } + else + { + long intervalMillis = Objects.requireNonNullElseGet(scheduleMetaData.getRepeatMillis(), () -> scheduleMetaData.getRepeatSeconds() * 1000); + scheduleBuilder = SimpleScheduleBuilder.simpleSchedule() + .withIntervalInMilliseconds(intervalMillis) + .repeatForever(); + } + + Date startAt = new Date(); + if(scheduleMetaData.getInitialDelayMillis() != null) + { + startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelayMillis()); + } + else if(scheduleMetaData.getInitialDelaySeconds() != null) + { + startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelaySeconds() * 1000); + } + else + { + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // by default, put a 3-second delay on everything we schedule // + // this gives us a chance to re-pause if the job was previously paused, but then we re-schedule it. // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + startAt.setTime(startAt.getTime() + 3000); + } + + /////////////////////////////////////// + // Define a Trigger for the schedule // + /////////////////////////////////////// + Trigger trigger = TriggerBuilder.newTrigger() + .withIdentity(new TriggerKey(schedulableIdentity.getIdentity(), groupName)) + .withDescription(schedulableIdentity.getDescription() + " - " + getScheduleDescriptionForTrigger(scheduleMetaData)) + .forJob(jobKey) + .withSchedule(scheduleBuilder) + // .startAt(startAt) + .build(); + + /////////////////////////////////////// + // Schedule the job with the trigger // + /////////////////////////////////////// + addOrReplaceJobAndTrigger(jobKey, jobDetail, trigger); + + /////////////////////////////////////////////////////////////////////////// + // if we're inside the setup event (e.g., initial startup), then capture // + // this job as one that is currently active and should be kept. // + /////////////////////////////////////////////////////////////////////////// + if(insideSetup) + { + scheduledJobsAtEndOfSetup.add(new QuartzJobAndTriggerWrapper(jobDetail, trigger, null)); + } + + return (true); + } + catch(Exception e) + { + LOG.warn("Error scheduling job", e, logPair("name", schedulableIdentity.getIdentity()), logPair("group", groupName)); + return (false); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getScheduleDescriptionForTrigger(QScheduleMetaData scheduleMetaData) + { + if(StringUtils.hasContent(scheduleMetaData.getDescription())) + { + return scheduleMetaData.getDescription(); + } + + if(StringUtils.hasContent(scheduleMetaData.getCronExpression())) + { + return "cron expression: " + scheduleMetaData.getCronExpression() + (StringUtils.hasContent(scheduleMetaData.getCronTimeZoneId()) ? " time zone: " + scheduleMetaData.getCronTimeZoneId() : ""); + } + + if(scheduleMetaData.getRepeatSeconds() != null) + { + return "repeat seconds: " + scheduleMetaData.getRepeatSeconds(); + } + + return ""; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unscheduleSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType) + { + //////////////////////////////////////////////////////////////////////////// + // only actually schedule things if we're past the server startup routine // + //////////////////////////////////////////////////////////////////////////// + if(!pastStartup) + { + return; + } + + deleteJob(new JobKey(schedulableIdentity.getIdentity(), schedulableType.getName())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unscheduleAll() throws QException + { + try + { + for(QuartzJobAndTriggerWrapper wrapper : queryQuartz()) + { + deleteJob(new JobKey(wrapper.jobDetail().getKey().getName(), wrapper.jobDetail().getKey().getGroup())); + } + } + catch(Exception e) + { + throw (new QException("Error unscheduling all quartz jobs", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void addOrReplaceJobAndTrigger(JobKey jobKey, JobDetail jobDetail, Trigger trigger) throws SchedulerException + { + boolean isJobAlreadyScheduled = isJobAlreadyScheduled(jobKey); + if(isJobAlreadyScheduled) + { + boolean wasPaused = wasExistingJobPaused(jobKey); + + this.scheduler.scheduleJob(jobDetail, Set.of(trigger), true); // note, true flag here replaces if already present. + LOG.info("Re-scheduled job", logPair("jobKey", jobKey)); + + if(wasPaused) + { + LOG.info("Re-pausing job", logPair("jobKey", jobKey)); + pauseJob(jobKey.getName(), jobKey.getGroup()); + } + } + else + { + this.scheduler.scheduleJob(jobDetail, trigger); + LOG.info("Scheduled new job", logPair("jobKey", jobKey)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean wasExistingJobPaused(JobKey jobKey) throws SchedulerException + { + List quartzJobAndTriggerWrappers = queryQuartz(); + Optional existingWrapper = quartzJobAndTriggerWrappers.stream().filter(w -> w.jobDetail().getKey().equals(jobKey)).findFirst(); + if(existingWrapper.isPresent()) + { + if(Trigger.TriggerState.PAUSED.equals(existingWrapper.get().triggerState())) + { + return (true); + } + } + + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean isJobAlreadyScheduled(JobKey jobKey) throws SchedulerException + { + Optional> jobGroupNames = jobGroupNamesMemoization.getResult(AnyKey.getInstance(), (x) -> scheduler.getJobGroupNames()); + if(jobGroupNames.isEmpty()) + { + throw (new SchedulerException("Error getting job group names")); + } + + for(String group : jobGroupNames.get()) + { + Optional> jobKeys = jobKeyNamesMemoization.getResult(group, (x) -> scheduler.getJobKeys(GroupMatcher.groupEquals(group))); + if(jobKeys.isEmpty()) + { + throw (new SchedulerException("Error getting job keys")); + } + + if(jobKeys.get().contains(jobKey)) + { + return (true); + } + } + + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean deleteJob(JobKey jobKey) + { + try + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UnscheduleJob.html // + // Deleting a Job and Unscheduling All of Its Triggers // + ///////////////////////////////////////////////////////////////////////////////////////////// + if(isJobAlreadyScheduled(jobKey)) + { + boolean result = scheduler.deleteJob(jobKey); + LOG.info("Attempted to delete quartz job", logPair("jobKey", jobKey), logPair("deleteJobResult", result)); + return (result); + } + + ///////////////////////////////////////// + // return true to indicate, we're good // + ///////////////////////////////////////// + LOG.info("Request to delete quartz job, but it is not already scheduled.", logPair("jobKey", jobKey)); + return (true); + } + catch(Exception e) + { + LOG.warn("Error deleting job", e, logPair("jobKey", jobKey)); + return false; + } + } + + + + /******************************************************************************* + ** Getter for qInstance + ** + *******************************************************************************/ + public QInstance getQInstance() + { + return qInstance; + } + + + + /******************************************************************************* + ** Getter for sessionSupplier + ** + *******************************************************************************/ + public Supplier getSessionSupplier() + { + return sessionSupplier; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void pauseAll() throws SchedulerException + { + /////////////////////////////////////////////////////////////////////////////// + // lesson from past self to future self: // + // pauseAll creates paused-group entries for all jobs - // + // and so they can only really be resumed by a resumeAll call... // + // even newly scheduled things become paused. Which can be quite confusing. // + // so, we don't want pause all. // + /////////////////////////////////////////////////////////////////////////////// + // this.scheduler.pauseAll(); + + List quartzJobAndTriggerWrappers = queryQuartz(); + for(QuartzJobAndTriggerWrapper wrapper : quartzJobAndTriggerWrappers) + { + this.pauseJob(wrapper.jobDetail().getKey().getName(), wrapper.jobDetail().getKey().getGroup()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void resumeAll() throws SchedulerException + { + ////////////////////////////////////////////////// + // this seems okay, even though pauseAll isn't. // + ////////////////////////////////////////////////// + this.scheduler.resumeAll(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void pauseJob(String jobName, String groupName) throws SchedulerException + { + LOG.info("Request to pause job", logPair("jobName", jobName)); + this.scheduler.pauseJob(new JobKey(jobName, groupName)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void resumeJob(String jobName, String groupName) throws SchedulerException + { + LOG.info("Request to resume job", logPair("jobName", jobName)); + this.scheduler.resumeJob(new JobKey(jobName, groupName)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + List queryQuartz() throws SchedulerException + { + return queryQuartzMemoization.getResultThrowing(AnyKey.getInstance(), (x) -> + { + List rs = new ArrayList<>(); + + for(String group : scheduler.getJobGroupNames()) + { + Set jobKeys = scheduler.getJobKeys(GroupMatcher.groupEquals(group)); + for(JobKey jobKey : jobKeys) + { + JobDetail jobDetail = scheduler.getJobDetail(jobKey); + List triggersOfJob = scheduler.getTriggersOfJob(jobKey); + for(Trigger trigger : triggersOfJob) + { + Trigger.TriggerState triggerState = scheduler.getTriggerState(trigger.getKey()); + rs.add(new QuartzJobAndTriggerWrapper(jobDetail, trigger, triggerState)); + } + } + } + + return (rs); + }).orElse(null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unInit() + { + /////////////////////////////////////////////////// + // resetting the singleton should be sufficient! // + /////////////////////////////////////////////////// + quartzScheduler = null; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java new file mode 100644 index 00000000..123b48c9 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** Manage process to pause all quartz jobs + *******************************************************************************/ +public class PauseAllQuartzJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Pause All Quartz Jobs") + .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to pause all quartz jobs."))), + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All quartz jobs have been paused"))))) + .withIcon(new QIcon("pause_circle_outline")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QuartzScheduler.getInstance().pauseAll(); + } + catch(Exception e) + { + throw (new QException("Error pausing all jobs", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java new file mode 100644 index 00000000..c5d3d38b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java @@ -0,0 +1,97 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.NoopTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class PauseQuartzJobsProcess extends AbstractLoadStep implements MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + String tableName = "quartzJobDetails"; + + return StreamedETLWithFrontendProcess.processMetaDataBuilder() + .withName(getClass().getSimpleName()) + .withLabel("Pause Quartz Jobs") + .withPreviewMessage("This is a preview of the jobs that will be paused.") + .withTableName(tableName) + .withSourceTable(tableName) + .withDestinationTable(tableName) + .withExtractStepClass(ExtractViaQueryStep.class) + .withTransformStepClass(NoopTransformStep.class) + .withLoadStepClass(getClass()) + .withIcon(new QIcon("pause_circle_outline")) + .withReviewStepRecordFields(List.of( + new QFieldMetaData("id", QFieldType.LONG), + new QFieldMetaData("jobName", QFieldType.STRING), + new QFieldMetaData("jobGroup", QFieldType.STRING))) + .getProcessMetaData(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QuartzScheduler instance = QuartzScheduler.getInstance(); + for(QRecord record : runBackendStepInput.getRecords()) + { + instance.pauseJob(record.getValueString("jobName"), record.getValueString("jobGroup")); + } + } + catch(Exception e) + { + throw (new QException("Error pausing jobs", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java new file mode 100644 index 00000000..7d81e31f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** Manage process to resume all quartz jobs + *******************************************************************************/ +public class ResumeAllQuartzJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Resume All Quartz Jobs") + .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to resume all quartz jobs."))), + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All quartz jobs have been resumed"))))) + .withIcon(new QIcon("play_circle_outline")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QuartzScheduler.getInstance().resumeAll(); + } + catch(Exception e) + { + throw (new QException("Error resuming all jobs", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java new file mode 100644 index 00000000..c4a8f3b8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java @@ -0,0 +1,97 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.NoopTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ResumeQuartzJobsProcess extends AbstractLoadStep implements MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + String tableName = "quartzJobDetails"; + + return StreamedETLWithFrontendProcess.processMetaDataBuilder() + .withName(getClass().getSimpleName()) + .withLabel("Resume Quartz Jobs") + .withPreviewMessage("This is a preview of the jobs that will be resumed.") + .withTableName(tableName) + .withSourceTable(tableName) + .withDestinationTable(tableName) + .withExtractStepClass(ExtractViaQueryStep.class) + .withTransformStepClass(NoopTransformStep.class) + .withLoadStepClass(getClass()) + .withIcon(new QIcon("play_circle_outline")) + .withReviewStepRecordFields(List.of( + new QFieldMetaData("id", QFieldType.LONG), + new QFieldMetaData("jobName", QFieldType.STRING), + new QFieldMetaData("jobGroup", QFieldType.STRING))) + .getProcessMetaData(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QuartzScheduler instance = QuartzScheduler.getInstance(); + for(QRecord record : runBackendStepInput.getRecords()) + { + instance.resumeJob(record.getValueString("jobName"), record.getValueString("jobGroup")); + } + } + catch(Exception e) + { + throw (new QException("Error resuming jobs", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDataPostQueryCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDataPostQueryCustomizer.java new file mode 100644 index 00000000..f16cbb83 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDataPostQueryCustomizer.java @@ -0,0 +1,106 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.tables; + + +import java.time.Instant; +import java.time.ZoneId; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.apache.commons.lang3.SerializationUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuartzJobDataPostQueryCustomizer extends AbstractPostQueryCustomizer +{ + private static final QLogger LOG = QLogger.getLogger(QuartzJobDataPostQueryCustomizer.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List apply(List records) + { + for(QRecord record : records) + { + if(record.getValue("jobData") != null) + { + try + { + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // this field has a blob of essentially a serialized map - so, deserialize that, then convert to JSON // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + byte[] value = record.getValueByteArray("jobData"); + if(value.length > 0) + { + Object deserialize = SerializationUtils.deserialize(value); + String json = JsonUtils.toJson(deserialize); + record.setValue("jobData", json); + } + } + catch(Exception e) + { + LOG.info("Error deserializing quartz job data", e); + } + } + + formatEpochTime(record, "nextFireTime"); + formatEpochTime(record, "prevFireTime"); + formatEpochTime(record, "startTime"); + } + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void formatEpochTime(QRecord record, String fieldName) + { + Long value = record.getValueLong(fieldName); + + try + { + if(value != null && value > 0) + { + Instant instant = Instant.ofEpochMilli(value); + record.setDisplayValue(fieldName, String.format("%,d", value) + " (" + QValueFormatter.formatDateTimeWithZone(instant.atZone(ZoneId.of(QContext.getQInstance().getDefaultTimeZoneId()))) + ")"); + } + } + catch(Exception e) + { + LOG.info("Error formatting an epoc time value", e, logPair("fieldName", fieldName), logPair("value", value)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/SchedulableType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/SchedulableType.java new file mode 100644 index 00000000..8f4d04c1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/SchedulableType.java @@ -0,0 +1,98 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable; + + +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SchedulableType +{ + private String name; + private QCodeReference runner; + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public SchedulableType withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for runner + *******************************************************************************/ + public QCodeReference getRunner() + { + return (this.runner); + } + + + + /******************************************************************************* + ** Setter for runner + *******************************************************************************/ + public void setRunner(QCodeReference runner) + { + this.runner = runner; + } + + + + /******************************************************************************* + ** Fluent setter for runner + *******************************************************************************/ + public SchedulableType withRunner(QCodeReference runner) + { + this.runner = runner; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/BasicSchedulableIdentity.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/BasicSchedulableIdentity.java new file mode 100644 index 00000000..2ad2a14d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/BasicSchedulableIdentity.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.identity; + + +import java.util.Objects; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** Basic implementation of interface for identifying schedulable things + *******************************************************************************/ +public class BasicSchedulableIdentity implements SchedulableIdentity +{ + private String identity; + private String description; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BasicSchedulableIdentity(String identity, String description) + { + if(!StringUtils.hasContent(identity)) + { + throw (new IllegalArgumentException("Identity may not be null or empty.")); + } + + this.identity = identity; + this.description = description; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean equals(Object o) + { + if(this == o) + { + return true; + } + + if(o == null || getClass() != o.getClass()) + { + return false; + } + + BasicSchedulableIdentity that = (BasicSchedulableIdentity) o; + return Objects.equals(identity, that.identity); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int hashCode() + { + return Objects.hash(identity); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getIdentity() + { + return identity; + } + + + + /******************************************************************************* + ** Getter for description + ** + *******************************************************************************/ + @Override + public String getDescription() + { + return description; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return getIdentity(); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentity.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentity.java new file mode 100644 index 00000000..f4cb1334 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentity.java @@ -0,0 +1,55 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.identity; + + +/******************************************************************************* + ** Unique identifier for a thing that can be scheduled + *******************************************************************************/ +public interface SchedulableIdentity +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Override + boolean equals(Object that); + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + int hashCode(); + + + /******************************************************************************* + ** + *******************************************************************************/ + String getIdentity(); + + + /******************************************************************************* + ** should NOT be part of equals & has code + *******************************************************************************/ + String getDescription(); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java new file mode 100644 index 00000000..0ad89408 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java @@ -0,0 +1,94 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.identity; + + +import java.util.HashMap; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; + + +/******************************************************************************* + ** Factory to produce SchedulableIdentity objects + *******************************************************************************/ +public class SchedulableIdentityFactory +{ + + /******************************************************************************* + ** Factory to create one of these for a scheduled job record + *******************************************************************************/ + public static BasicSchedulableIdentity of(ScheduledJob scheduledJob) + { + String description = ""; + SchedulableType schedulableType = QContext.getQInstance().getSchedulableType(scheduledJob.getType()); + if(schedulableType != null) + { + try + { + SchedulableRunner runner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + description = runner.getDescription(new HashMap<>(scheduledJob.getJobParametersMap())); + } + catch(Exception e) + { + description = "type: " + schedulableType.getName(); + } + } + + return new BasicSchedulableIdentity("scheduledJob:" + scheduledJob.getId(), description); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static BasicSchedulableIdentity of(QProcessMetaData process) + { + return new BasicSchedulableIdentity("process:" + process.getName(), "Process: " + process.getName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static SchedulableIdentity of(QQueueMetaData queue) + { + return new BasicSchedulableIdentity("queue:" + queue.getName(), "Queue: " + queue.getName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static SchedulableIdentity of(PollingAutomationPerTableRunner.TableActionsInterface tableActions) + { + return new BasicSchedulableIdentity("tableAutomations:" + tableActions.tableName() + "." + tableActions.status(), "TableAutomations: " + tableActions.tableName() + "." + tableActions.status()); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableProcessRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableProcessRunner.java new file mode 100644 index 00000000..c94d25ba --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableProcessRunner.java @@ -0,0 +1,188 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.runner; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Schedulable process runner - e.g., how a QProcess is run by a scheduler. + *******************************************************************************/ +public class SchedulableProcessRunner implements SchedulableRunner +{ + private static final QLogger LOG = QLogger.getLogger(SchedulableProcessRunner.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(Map params) + { + String processName = ValueUtils.getValueAsString(params.get("processName")); + + /////////////////////////////////////// + // get the process from the instance // + /////////////////////////////////////// + QInstance qInstance = QContext.getQInstance(); + QProcessMetaData process = qInstance.getProcess(processName); + if(process == null) + { + LOG.warn("Could not find scheduled process in QInstance", logPair("processName", processName)); + return; + } + + /////////////////////////////////////////////// + // if the job has variant data, get it ready // + /////////////////////////////////////////////// + Map backendVariantData = null; + if(params.containsKey("backendVariantData")) + { + backendVariantData = (Map) params.get("backendVariantData"); + } + + Map processInputValues = buildProcessInputValuesMap(params, process); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running scheduled process", logPair("processName", processName)); + SchedulerUtils.runProcess(qInstance, () -> QContext.getQSession(), qInstance.getProcess(processName), backendVariantData, processInputValues); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void validateParams(SchedulableIdentity schedulableIdentity, Map paramMap) throws QException + { + String processName = ValueUtils.getValueAsString(paramMap.get("processName")); + if(!StringUtils.hasContent(processName)) + { + throw (new QException("Missing scheduledJobParameter with key [processName] in " + schedulableIdentity)); + } + + QProcessMetaData process = QContext.getQInstance().getProcess(processName); + if(process == null) + { + throw (new QException("Unrecognized processName [" + processName + "] in " + schedulableIdentity)); + } + + if(process.getSchedule() != null) + { + throw (new QException("Process [" + processName + "] has a schedule in its metaData - so it should not be dynamically scheduled via a scheduled job! " + schedulableIdentity)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDescription(Map params) + { + return "Process: " + params.get("processName"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Map buildProcessInputValuesMap(Map params, QProcessMetaData process) + { + Map processInputValues = new HashMap<>(); + + ////////////////////////////////////////////////////////////////////////////////////// + // track which keys need processed - start by removing ones we know we handle above // + ////////////////////////////////////////////////////////////////////////////////////// + Set keys = new HashSet<>(params.keySet()); + keys.remove("processName"); + keys.remove("backendVariantData"); + + if(!keys.isEmpty()) + { + ////////////////////////////////////////////////////////////////////////// + // first make a pass going over the process's identified input fields - // + // getting values from the quartz job data map, and putting them into // + // the process input value map as the field's type (if we can) // + ////////////////////////////////////////////////////////////////////////// + for(QFieldMetaData inputField : process.getInputFields()) + { + String fieldName = inputField.getName(); + if(params.containsKey(fieldName)) + { + Object value = params.get(fieldName); + try + { + processInputValues.put(fieldName, ValueUtils.getValueAsFieldType(inputField.getType(), value)); + keys.remove(fieldName); + } + catch(Exception e) + { + LOG.warn("Error getting process input value from quartz job data map", e, logPair("fieldName", fieldName), logPair("value", value)); + } + } + } + + //////////////////////////////////////////////////////////////////////////////////////// + // if any values are left in the map (based on keys set that we're removing from) // + // then try to put those in the input map (assuming they can be cast to Serializable) // + //////////////////////////////////////////////////////////////////////////////////////// + for(String key : keys) + { + Object value = params.get(key); + try + { + processInputValues.put(key, (Serializable) value); + } + catch(Exception e) + { + LOG.warn("Error getting process input value from quartz job data map", e, logPair("key", key), logPair("value", value)); + } + } + } + + return processInputValues; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableRunner.java new file mode 100644 index 00000000..b718d609 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableRunner.java @@ -0,0 +1,51 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.runner; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; + + +/******************************************************************************* + ** Interface for different types of schedulabe things that can be run + *******************************************************************************/ +public interface SchedulableRunner +{ + + /******************************************************************************* + ** + *******************************************************************************/ + void run(Map params); + + + /******************************************************************************* + ** + *******************************************************************************/ + void validateParams(SchedulableIdentity schedulableIdentity, Map paramMap) throws QException; + + /******************************************************************************* + ** + *******************************************************************************/ + String getDescription(Map params); +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableSQSQueueRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableSQSQueueRunner.java new file mode 100644 index 00000000..f2cc4dab --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableSQSQueueRunner.java @@ -0,0 +1,135 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.runner; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Schedulable SQSQueue runner - e.g., how an SQSQueuePoller is run by a scheduler. + *******************************************************************************/ +public class SchedulableSQSQueueRunner implements SchedulableRunner +{ + private static final QLogger LOG = QLogger.getLogger(SchedulableSQSQueueRunner.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(Map params) + { + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + + String queueName = ValueUtils.getValueAsString(params.get("queueName")); + if(!StringUtils.hasContent(queueName)) + { + LOG.warn("Missing queueName in params."); + return; + } + + QQueueMetaData queue = qInstance.getQueue(queueName); + if(queue == null) + { + LOG.warn("Unrecognized queueName [" + queueName + "]"); + return; + } + + QQueueProviderMetaData queueProvider = qInstance.getQueueProvider(queue.getProviderName()); + if(!(queueProvider instanceof SQSQueueProviderMetaData)) + { + LOG.warn("Queue [" + queueName + "] is of an unsupported queue provider type (not SQS)"); + return; + } + + SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); + sqsQueuePoller.setQueueMetaData(queue); + sqsQueuePoller.setQueueProviderMetaData((SQSQueueProviderMetaData) queueProvider); + sqsQueuePoller.setQInstance(qInstance); + sqsQueuePoller.setSessionSupplier(QuartzScheduler.getInstance().getSessionSupplier()); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running SQS Queue poller", logPair("queueName", queueName)); + sqsQueuePoller.run(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void validateParams(SchedulableIdentity schedulableIdentity, Map paramMap) throws QException + { + String queueName = ValueUtils.getValueAsString(paramMap.get("queueName")); + if(!StringUtils.hasContent(queueName)) + { + throw (new QException("Missing scheduledJobParameter with key [queueName] in " + schedulableIdentity)); + } + + QQueueMetaData queue = QContext.getQInstance().getQueue(queueName); + if(queue == null) + { + throw (new QException("Unrecognized queueName [" + queueName + "] in " + schedulableIdentity)); + } + + QQueueProviderMetaData queueProvider = QContext.getQInstance().getQueueProvider(queue.getProviderName()); + if(!(queueProvider instanceof SQSQueueProviderMetaData)) + { + throw (new QException("Queue [" + queueName + "] is of an unsupported queue provider type (not SQS) in " + schedulableIdentity)); + } + + if(queue.getSchedule() != null) + { + throw (new QException("Queue [" + queueName + "] has a schedule in its metaData - so it should not be dynamically scheduled via a scheduled job! " + schedulableIdentity)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDescription(Map params) + { + return "Queue: " + params.get("queueName"); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java new file mode 100644 index 00000000..44bd2f20 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java @@ -0,0 +1,164 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.runner; + + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Schedulable TableAutomations runner - e.g., how a table automations are run + ** by a scheduler. + *******************************************************************************/ +public class SchedulableTableAutomationsRunner implements SchedulableRunner +{ + private static final QLogger LOG = QLogger.getLogger(SchedulableTableAutomationsRunner.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(Map params) + { + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + + String tableName = ValueUtils.getValueAsString(params.get("tableName")); + if(!StringUtils.hasContent(tableName)) + { + LOG.warn("Missing tableName in params."); + return; + } + + QTableMetaData table = qInstance.getTable(tableName); + if(table == null) + { + LOG.warn("Unrecognized tableName [" + tableName + "]"); + return; + } + + AutomationStatus automationStatus = AutomationStatus.valueOf(ValueUtils.getValueAsString(params.get("automationStatus"))); + + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + if(automationDetails == null) + { + LOG.warn("Could not find automationDetails for table for automations in QInstance", logPair("tableName", tableName)); + return; + } + + /////////////////////////////////// + // todo - sharded automations... // + /////////////////////////////////// + PollingAutomationPerTableRunner.TableActionsInterface tableAction = new PollingAutomationPerTableRunner.TableActions(tableName, automationDetails, automationStatus); + PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationDetails.getProviderName(), QuartzScheduler.getInstance().getSessionSupplier(), tableAction); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running Table Automations", logPair("tableName", tableName), logPair("automationStatus", automationStatus)); + runner.run(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void validateParams(SchedulableIdentity schedulableIdentity, Map paramMap) throws QException + { + String tableName = ValueUtils.getValueAsString(paramMap.get("tableName")); + if(!StringUtils.hasContent(tableName)) + { + throw (new QException("Missing scheduledJobParameter with key [tableName] in " + schedulableIdentity)); + } + + String automationStatus = ValueUtils.getValueAsString(paramMap.get("automationStatus")); + if(!StringUtils.hasContent(automationStatus)) + { + throw (new QException("Missing scheduledJobParameter with key [automationStatus] in " + schedulableIdentity)); + } + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + if(table == null) + { + throw (new QException("Unrecognized tableName [" + tableName + "] in " + schedulableIdentity)); + } + + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + if(automationDetails == null) + { + throw (new QException("Table [" + tableName + "] does not have automationDetails in " + schedulableIdentity)); + } + + if(automationDetails.getSchedule() != null) + { + throw (new QException("Table [" + tableName + "] automationDetails has a schedule in its metaData - so it should not be dynamically scheduled via a scheduled job! " + schedulableIdentity)); + } + + QAutomationProviderMetaData automationProvider = QContext.getQInstance().getAutomationProvider(automationDetails.getProviderName()); + + List tableActionList = PollingAutomationPerTableRunner.getTableActions(QContext.getQInstance(), automationProvider.getName()); + for(PollingAutomationPerTableRunner.TableActionsInterface tableActions : tableActionList) + { + if(tableActions.status().name().equals(automationStatus)) + { + return; + } + } + + ///////////////////////////////////////////////////////////////////////////////////// + // if we get out of the loop, it means we didn't find a matching status - so throw // + ///////////////////////////////////////////////////////////////////////////////////// + throw (new QException("Did not find table automation actions matching automationStatus [" + automationStatus + "] for table [" + tableName + "] in " + schedulableIdentity + + " (Found: " + tableActionList.stream().map(ta -> ta.status().name()).collect(Collectors.joining(",")) + ")")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDescription(Map params) + { + return "TableAutomations: " + params.get("tableName") + "." + params.get("automationStatus"); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java new file mode 100644 index 00000000..bbdf6466 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java @@ -0,0 +1,87 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.simple; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.context.CapturedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SimpleJobRunner implements Runnable +{ + private static final QLogger LOG = QLogger.getLogger(SimpleJobRunner.class); + + private QInstance qInstance; + private SchedulableType schedulableType; + private Map params; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SimpleJobRunner(QInstance qInstance, SchedulableType type, Map params) + { + this.qInstance = qInstance; + this.schedulableType = type; + this.params = params; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run() + { + CapturedContext capturedContext = QContext.capture(); + try + { + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); + QContext.init(qInstance, simpleScheduler.getSessionSupplier().get()); + + SchedulableRunner schedulableRunner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + schedulableRunner.run(params); + } + catch(Exception e) + { + LOG.warn("Error running SimpleScheduler job", e, logPair("params", params)); + } + finally + { + QContext.init(capturedContext); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java new file mode 100644 index 00000000..04ff9a6f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java @@ -0,0 +1,299 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.simple; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** QQQ Service (Singleton) that starts up repeating, scheduled jobs within QQQ. + ** + ** These include: + ** - Automation providers (which require polling) + ** - Queue pollers + ** - Scheduled processes. + ** + ** All of these jobs run using a "system session" - as defined by the sessionSupplier. + *******************************************************************************/ +public class SimpleScheduler implements QSchedulerInterface +{ + private static final QLogger LOG = QLogger.getLogger(SimpleScheduler.class); + + private static SimpleScheduler simpleScheduler = null; + private final QInstance qInstance; + private String schedulerName; + + protected Supplier sessionSupplier; + + ///////////////////////////////////////////////////////////////////////////////////// + // for jobs that don't define a delay index, auto-stagger them, using this counter // + ///////////////////////////////////////////////////////////////////////////////////// + private int delayIndex = 0; + + private Map executors = new LinkedHashMap<>(); + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private SimpleScheduler(QInstance qInstance) + { + this.qInstance = qInstance; + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static SimpleScheduler getInstance(QInstance qInstance) + { + if(simpleScheduler == null) + { + simpleScheduler = new SimpleScheduler(qInstance); + } + return (simpleScheduler); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void start() + { + for(StandardScheduledExecutor executor : executors.values()) + { + executor.start(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stopAsync() + { + for(StandardScheduledExecutor scheduledExecutor : executors.values()) + { + scheduledExecutor.stopAsync(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stop() + { + for(StandardScheduledExecutor scheduledExecutor : executors.values()) + { + scheduledExecutor.stop(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setupSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType, Map parameters, QScheduleMetaData schedule, boolean allowedToStart) + { + if(!allowedToStart) + { + return; + } + + SimpleJobRunner simpleJobRunner = new SimpleJobRunner(qInstance, schedulableType, new HashMap<>(parameters)); + StandardScheduledExecutor executor = new StandardScheduledExecutor(simpleJobRunner); + executor.setName(schedulableIdentity.getIdentity()); + setScheduleInExecutor(schedule, executor); + executors.put(schedulableIdentity, executor); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setScheduleInExecutor(QScheduleMetaData schedule, StandardScheduledExecutor executor) + { + if(schedule.getRepeatMillis() != null) + { + executor.setDelayMillis(schedule.getRepeatMillis()); + } + else + { + executor.setDelayMillis(1000 * schedule.getRepeatSeconds()); + } + + if(schedule.getInitialDelayMillis() != null) + { + executor.setInitialDelayMillis(schedule.getInitialDelayMillis()); + } + else if(schedule.getInitialDelaySeconds() != null) + { + executor.setInitialDelayMillis(1000 * schedule.getInitialDelaySeconds()); + } + else + { + executor.setInitialDelayMillis(1000 * ++delayIndex); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unscheduleSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType) + { + StandardScheduledExecutor executor = executors.get(schedulableIdentity); + if(executor != null) + { + LOG.info("Stopping job in simple scheduler", logPair("identity", schedulableIdentity)); + executors.remove(schedulableIdentity); + executor.stop(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unscheduleAll() throws QException + { + for(Map.Entry entry : new HashSet<>(executors.entrySet())) + { + StandardScheduledExecutor executor = executors.remove(entry.getKey()); + if(executor != null) + { + executor.stopAsync(); + } + } + } + + + + /******************************************************************************* + ** Setter for sessionSupplier + ** + *******************************************************************************/ + public void setSessionSupplier(Supplier sessionSupplier) + { + this.sessionSupplier = sessionSupplier; + } + + + + /******************************************************************************* + ** Getter for sessionSupplier + ** + *******************************************************************************/ + public Supplier getSessionSupplier() + { + return sessionSupplier; + } + + + + /******************************************************************************* + ** Getter for managedExecutors + ** + *******************************************************************************/ + public List getExecutors() + { + return new ArrayList<>(executors.values()); + } + + + + /******************************************************************************* + ** Getter for schedulerName + *******************************************************************************/ + public String getSchedulerName() + { + return (this.schedulerName); + } + + + + /******************************************************************************* + ** Setter for schedulerName + *******************************************************************************/ + public void setSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + } + + + + /******************************************************************************* + ** Fluent setter for schedulerName + *******************************************************************************/ + public SimpleScheduler withSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unInit() + { + ////////////////////////////////////////////////// + // resetting the singleton should be sufficient // + ////////////////////////////////////////////////// + simpleScheduler = null; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java similarity index 95% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java index 3910e65f..36c858f5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.scheduler; +package com.kingsrook.qqq.backend.core.scheduler.simple; import java.util.concurrent.Executors; @@ -32,8 +32,10 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; /******************************************************************************* - ** Standard class ran by ScheduleManager. Takes a Runnable in its constructor - - ** that's the code that actually executes. + ** Standard class ran by SimpleScheduler. Takes a Runnable in its constructor - + ** that's the code that actually executes. Internally, this class will launch + ** a newSingleThreadScheduledExecutor / ScheduledExecutorService to run the + ** runnable on a repeating delay. ** *******************************************************************************/ public class StandardScheduledExecutor diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java index 95b8d091..1e53c346 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java @@ -518,6 +518,24 @@ public class CollectionUtils + /******************************************************************************* + ** Convert a collection of QRecords to a map, from one field's values out of + ** those records, to the records themselves. + *******************************************************************************/ + public static Map recordsToMap(Collection records, String keyFieldName, Class type) + { + Map rs = new HashMap<>(); + + for(QRecord record : nonNullCollection(records)) + { + rs.put(ValueUtils.getValueAsType(type, record.getValue(keyFieldName)), record); + } + + return (rs); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java index a2a36a37..e7d8e9d3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java @@ -362,6 +362,7 @@ public class JsonUtils switch(metaData.getType()) { case INTEGER -> record.setValue(fieldName, jsonObjectToUse.optInt(backendName)); + case LONG -> record.setValue(fieldName, jsonObjectToUse.optLong(backendName)); case DECIMAL -> record.setValue(fieldName, jsonObjectToUse.optBigDecimal(backendName, null)); case BOOLEAN -> record.setValue(fieldName, jsonObjectToUse.optBoolean(backendName)); case DATE_TIME -> diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index 8bc4553a..9c31b8a5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -114,6 +114,113 @@ public class ValueUtils + /******************************************************************************* + ** Type-safely make an Long from any Object. + ** null and empty-string inputs return null. + ** We try to strip away commas and decimals (as long as they are exactly equal to the int value) + ** We may throw if the input can't be converted to an integer. + *******************************************************************************/ + public static Long getValueAsLong(Object value) throws QValueException + { + try + { + if(value == null) + { + return (null); + } + else if(value instanceof Integer i) + { + return Long.valueOf((i)); + } + else if(value instanceof Long l) + { + return (l); + } + else if(value instanceof BigInteger b) + { + return (b.longValue()); + } + else if(value instanceof Float f) + { + if(f.longValue() != f) + { + throw (new QValueException(f + " does not have an exact integer representation.")); + } + return (f.longValue()); + } + else if(value instanceof Double d) + { + if(d.longValue() != d) + { + throw (new QValueException(d + " does not have an exact integer representation.")); + } + return (d.longValue()); + } + else if(value instanceof BigDecimal bd) + { + return bd.longValueExact(); + } + else if(value instanceof PossibleValueEnum pve) + { + return getValueAsLong(pve.getPossibleValueId()); + } + else if(value instanceof String s) + { + if(!StringUtils.hasContent(s)) + { + return (null); + } + + try + { + return (Long.parseLong(s)); + } + catch(NumberFormatException nfe) + { + if(s.contains(",")) + { + String sWithoutCommas = s.replaceAll(",", ""); + try + { + return (getValueAsLong(sWithoutCommas)); + } + catch(Exception ignore) + { + throw (nfe); + } + } + if(s.matches(".*\\.\\d+$")) + { + String sWithoutDecimal = s.replaceAll("\\.\\d+$", ""); + try + { + return (getValueAsLong(sWithoutDecimal)); + } + catch(Exception ignore) + { + throw (nfe); + } + } + throw (nfe); + } + } + else + { + throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to Long.")); + } + } + catch(QValueException qve) + { + throw (qve); + } + catch(Exception e) + { + throw (new QValueException("Value [" + value + "] could not be converted to a Long.", e)); + } + } + + + /******************************************************************************* ** Type-safely make an Integer from any Object. ** null and empty-string inputs return null. @@ -693,6 +800,7 @@ public class ValueUtils { case STRING, TEXT, HTML, PASSWORD -> getValueAsString(value); case INTEGER -> getValueAsInteger(value); + case LONG -> getValueAsLong(value); case DECIMAL -> getValueAsBigDecimal(value); case BOOLEAN -> getValueAsBoolean(value); case DATE -> getValueAsLocalDate(value); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/YamlUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/YamlUtils.java index 1c7a8f53..c52bf027 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/YamlUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/YamlUtils.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.utils; import java.util.Map; +import java.util.function.Consumer; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -56,6 +57,16 @@ public class YamlUtils ** *******************************************************************************/ public static String toYaml(Object object) + { + return toYaml(object, null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String toYaml(Object object, Consumer objectMapperCustomizer) { try { @@ -66,7 +77,10 @@ public class YamlUtils objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); - // todo? objectMapper.setFilterProvider(new OmitDefaultValuesFilterProvider()); + if(objectMapperCustomizer != null) + { + objectMapperCustomizer.accept(objectMapper); + } objectMapper.findAndRegisterModules(); return (objectMapper.writeValueAsString(object)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java new file mode 100644 index 00000000..bcf1862b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java @@ -0,0 +1,135 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +import java.math.BigDecimal; + + +/******************************************************************************* + ** Long version of data aggregator + *******************************************************************************/ +public class LongAggregates implements AggregatesInterface +{ + private int count = 0; + // private Long countDistinct; + private Long sum; + private Long min; + private Long max; + + + + /******************************************************************************* + ** Add a new value to this aggregate set + *******************************************************************************/ + public void add(Long input) + { + if(input == null) + { + return; + } + + count++; + + if(sum == null) + { + sum = input; + } + else + { + sum = sum + input; + } + + if(min == null || input < min) + { + min = input; + } + + if(max == null || input > max) + { + max = input; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int getCount() + { + return (count); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Long getSum() + { + return (sum); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Long getMin() + { + return (min); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Long getMax() + { + return (max); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getAverage() + { + if(this.count > 0) + { + return (BigDecimal.valueOf(this.sum.doubleValue() / (double) this.count)); + } + else + { + return (null); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/lambdas/UnsafeVoidVoidMethod.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/lambdas/UnsafeVoidVoidMethod.java new file mode 100644 index 00000000..e448d2e9 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/lambdas/UnsafeVoidVoidMethod.java @@ -0,0 +1,37 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.lambdas; + + +/******************************************************************************* + ** + *******************************************************************************/ +@FunctionalInterface +public interface UnsafeVoidVoidMethod +{ + + /******************************************************************************* + ** + *******************************************************************************/ + void run() throws T; + +} diff --git a/qqq-backend-core/src/main/resources/log4j2.xml b/qqq-backend-core/src/main/resources/log4j2.xml index 349d47d1..ffb3baa6 100644 --- a/qqq-backend-core/src/main/resources/log4j2.xml +++ b/qqq-backend-core/src/main/resources/log4j2.xml @@ -26,6 +26,11 @@ + + + + + diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java index dc276b0d..c4e47381 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java @@ -575,8 +575,8 @@ class PollingAutomationPerTableRunnerTest extends BaseTest @Test void testLoadingRecordTypesToEnsureClassCoverage() { - new PollingAutomationPerTableRunner.TableActions(null, null).noopToFakeTestCoverage(); - new PollingAutomationPerTableRunner.ShardedTableActions(null, null, null, null, null).noopToFakeTestCoverage(); + new PollingAutomationPerTableRunner.TableActions(null, null, null).noopToFakeTestCoverage(); + new PollingAutomationPerTableRunner.ShardedTableActions(null, null, null, null, null, null).noopToFakeTestCoverage(); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java index 57e0055d..e08b4ac1 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java @@ -43,7 +43,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; -import com.kingsrook.qqq.backend.core.scheduler.StandardScheduledExecutor; +import com.kingsrook.qqq.backend.core.scheduler.simple.StandardScheduledExecutor; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoaderTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoaderTest.java new file mode 100644 index 00000000..83cf8298 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoaderTest.java @@ -0,0 +1,94 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.customizers; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.utils.Timer; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for QCodeLoader + *******************************************************************************/ +class QCodeLoaderTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetAdHoc() + { + QCodeLoader qCodeLoader = QCodeLoader.getAdHoc(QCodeLoader.class, new QCodeReference(QCodeLoader.class)); + assertThat(qCodeLoader).isInstanceOf(QCodeLoader.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + @Disabled("performance test, used during memoization change") + void testBulkPerformance() + { + Timer timer = new Timer("start"); + for(int i = 0; i < 5; i++) + { + useCodeLoader(1_000_000); + timer.mark("done with code loader"); + + useNew(1_000_000); + timer.mark("done with new"); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void useNew(int count) + { + for(int i = 0; i < count; i++) + { + QCodeLoader qCodeLoader = new QCodeLoader(); + } + } + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void useCodeLoader(int count) + { + for(int i = 0; i < count; i++) + { + QCodeLoader qCodeLoader = QCodeLoader.getAdHoc(QCodeLoader.class, new QCodeReference(QCodeLoader.class)); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java index f012677d..839172d8 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java @@ -103,6 +103,7 @@ public class RunBackendStepActionTest extends BaseTest { case STRING -> "ABC"; case INTEGER -> 42; + case LONG -> 42L; case DECIMAL -> new BigDecimal("47"); case BOOLEAN -> true; case DATE, TIME, DATE_TIME -> null; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java index e17d6ed0..bbb5493b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java @@ -26,15 +26,21 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; @@ -777,4 +783,142 @@ class InsertActionTest extends BaseTest assertEquals(2, records.get(1).getValueInteger("noOfShoes")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCustomizers() throws QException + { + String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY; + + { + QContext.getQInstance().getTable(tableName).withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(TestPreInsertCustomizer.class)); + + List records = new InsertAction().execute(new InsertInput(tableName) + .withRecord(new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff"))) + .getRecords(); + assertEquals(1701, records.get(0).getValueInteger("noOfShoes")); + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // because this was a pre-action, the value should actually be inserted - so re-query and get it // + /////////////////////////////////////////////////////////////////////////////////////////////////// + assertEquals(1701, new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(1)).getValueInteger("noOfShoes")); + + QContext.getQInstance().getTable(tableName).withCustomizers(new HashMap<>()); + } + + { + QContext.getQInstance().getTable(tableName).withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(TestPostInsertCustomizer.class)); + + List records = new InsertAction().execute(new InsertInput(tableName) + .withRecord(new QRecord().withValue("firstName", "Thom").withValue("lastName", "Chutterloin"))) + .getRecords(); + assertEquals(47, records.get(0).getValueInteger("homeStateId")); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // because this was a post-action, the value should NOT actually be inserted - so re-query and confirm null // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertNull(new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(2)).getValueInteger("homeStateId")); + + QContext.getQInstance().getTable(tableName).withCustomizers(new HashMap<>()); + } + + { + QContext.getQInstance().getTable(tableName).withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(TestTableCustomizer.class)); + QContext.getQInstance().getTable(tableName).withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(TestTableCustomizer.class)); + + List records = new InsertAction().execute(new InsertInput(tableName) + .withRecord(new QRecord().withValue("firstName", "Thom").withValue("lastName", "Chutterloin"))) + .getRecords(); + assertEquals(1701, records.get(0).getValueInteger("noOfShoes")); + assertEquals(47, records.get(0).getValueInteger("homeStateId")); + + ////////////////////////////////////////////////////////////////////// + // merger of the two above - one pre, one post, so one set, one not // + ////////////////////////////////////////////////////////////////////// + QRecord fetchedRecord = new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(2)); + assertEquals(1701, records.get(0).getValueInteger("noOfShoes")); + assertNull(fetchedRecord.getValueInteger("homeStateId")); + + QContext.getQInstance().getTable(tableName).withCustomizers(new HashMap<>()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestPreInsertCustomizer extends AbstractPreInsertCustomizer + { + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List apply(List records) throws QException + { + List rs = new ArrayList<>(); + records.forEach(r -> rs.add(new QRecord(r).withValue("noOfShoes", 1701))); + return rs; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestPostInsertCustomizer extends AbstractPostInsertCustomizer + { + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List apply(List records) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // grr, memory backend let's make sure to return a clone (so we don't edit what's stored!) // + ///////////////////////////////////////////////////////////////////////////////////////////// + List rs = new ArrayList<>(); + records.forEach(r -> rs.add(new QRecord(r).withValue("homeStateId", 47))); + return rs; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestTableCustomizer implements TableCustomizerInterface + { + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + List rs = new ArrayList<>(); + records.forEach(r -> rs.add(new QRecord(r).withValue("noOfShoes", 1701))); + return rs; + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // grr, memory backend let's make sure to return a clone (so we don't edit what's stored!) // + ///////////////////////////////////////////////////////////////////////////////////////////// + List rs = new ArrayList<>(); + records.forEach(r -> rs.add(new QRecord(r).withValue("homeStateId", 47))); + return rs; + } + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java index 53e9ec96..a4ab55c9 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java @@ -232,6 +232,23 @@ class QInstanceEnricherTest extends BaseTest } + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testInferNameFromBackendName() + { + assertEquals("id", QInstanceEnricher.inferNameFromBackendName("id")); + assertEquals("wordAnotherWordMoreWords", QInstanceEnricher.inferNameFromBackendName("word_another_word_more_words")); + assertEquals("lUlUlUl", QInstanceEnricher.inferNameFromBackendName("l_ul_ul_ul")); + assertEquals("tlaFirst", QInstanceEnricher.inferNameFromBackendName("tla_first")); + assertEquals("wordThenTlaInMiddle", QInstanceEnricher.inferNameFromBackendName("word_then_tla_in_middle")); + assertEquals("endWithTla", QInstanceEnricher.inferNameFromBackendName("end_with_tla")); + assertEquals("tlaAndAnotherTla", QInstanceEnricher.inferNameFromBackendName("tla_and_another_tla")); + assertEquals("allCaps", QInstanceEnricher.inferNameFromBackendName("ALL_CAPS")); + } + + /******************************************************************************* ** diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 18786a09..0ad55dde 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; @@ -70,6 +71,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMeta import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; @@ -368,6 +370,127 @@ public class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test_validateSchedules() + { + String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE; + Supplier baseScheduleMetaData = () -> new QScheduleMetaData() + .withSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME); + + //////////////////////////////////////////////////// + // do our basic schedule validations on a process // + //////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()), + "either repeatMillis or repeatSeconds or cronExpression must be set"); + + String validCronString = "* * * * * ?"; + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "both a repeat time and cronExpression may not be set"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatSeconds(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "both a repeat time and cronExpression may not be set"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatSeconds(1) + .withRepeatMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "both a repeat time and cronExpression may not be set"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withInitialDelaySeconds(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "cron schedule may not have an initial delay"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withInitialDelayMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "cron schedule may not have an initial delay"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withInitialDelaySeconds(1) + .withInitialDelayMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "cron schedule may not have an initial delay"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression(validCronString)), + "must specify a cronTimeZoneId"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression(validCronString) + .withCronTimeZoneId("foobar")), + "unrecognized cronTimeZoneId: foobar"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression("* * * * * *") + .withCronTimeZoneId("UTC")), + "invalid cron expression: Support for specifying both"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression("x") + .withCronTimeZoneId("UTC")), + "invalid cron expression: Illegal cron expression format"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatSeconds(10) + .withCronTimeZoneId("UTC")), + "non-cron schedule must not specify a cronTimeZoneId"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withSchedulerName(null) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "is missing a scheduler name"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withSchedulerName("not-a-scheduler") + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "referencing an unknown scheduler name: not-a-scheduler"); + + ///////////////////////////////// + // validate some success cases // + ///////////////////////////////// + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withRepeatSeconds(1))); + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withRepeatMillis(1))); + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("UTC"))); + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("America/New_York"))); + + /////////////////////////////////////////////////////////////// + // make sure table automations get their schedules validated // + /////////////////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().withSchedule(baseScheduleMetaData.get() + .withSchedulerName(null) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "is missing a scheduler name"); + + //////////////////////////////////////////////////// + // make sure queues get their schedules validated // + //////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> (qInstance.getQueue(TestUtils.TEST_SQS_QUEUE)).withSchedule(baseScheduleMetaData.get() + .withSchedulerName(null) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "is missing a scheduler name"); + + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -387,7 +510,7 @@ public class QInstanceValidatorTest extends BaseTest finally { QInstanceValidator.removeAllValidatorPlugins(); - + //////////////////////////////////////////////////// // make sure if remove all plugins, we don't fail // //////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java new file mode 100644 index 00000000..7e926fe1 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java @@ -0,0 +1,383 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers; + + +import java.time.Instant; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobParameter; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobsMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; +import org.assertj.core.api.AbstractStringAssert; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.quartz.CronTrigger; +import org.quartz.SchedulerException; +import org.quartz.SimpleTrigger; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for ScheduledJobTableCustomizer + *******************************************************************************/ +class ScheduledJobTableCustomizerTest extends BaseTest +{ + private static final String GOOD_CRON = "0 * * * * ?"; + private static final String GOOD_CRON_2 = "* * * * * ?"; + private static final String BAD_CRON = "* * * * * *"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws QException + { + QInstance qInstance = QContext.getQInstance(); + QuartzTestUtils.setupInstanceForQuartzTests(); + + QSession qSession = QContext.getQSession(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); + qScheduleManager.start(); + + new ScheduledJobsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QuartzTestUtils.afterEach(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPreInsertAssertValidationErrors() throws QException + { + UnsafeFunction, QRecord, QException> tryToInsert = consumer -> + { + ScheduledJob scheduledJob = new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true); + consumer.accept(scheduledJob); + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(scheduledJob)); + return (insertOutput.getRecords().get(0)); + }; + + ///////////////////////////////////////////////////////// + // lambdas to run a test and assert about no of errors // + ///////////////////////////////////////////////////////// + Function> assertOneErrorExtractingMessage = qRecord -> assertThat(qRecord.getErrors()).hasSize(1).first().extracting("message").asString(); + Consumer assertNoErrors = qRecord -> assertThat(qRecord.getErrors()).hasSize(0); + + assertOneErrorExtractingMessage.apply(tryToInsert.apply(sj -> sj.setId(null))) + .contains("Either Cron Expression or Repeat Seconds must be given"); + + assertOneErrorExtractingMessage.apply(tryToInsert.apply(sj -> sj.withRepeatSeconds(1).withCronExpression(GOOD_CRON).withCronTimeZoneId("UTC"))) + .contains("Cron Expression and Repeat Seconds may not both be given"); + + assertOneErrorExtractingMessage.apply(tryToInsert.apply(sj -> sj.withRepeatSeconds(null).withCronExpression(GOOD_CRON))) + .contains("If a Cron Expression is given, then a Cron Time Zone is required"); + + assertOneErrorExtractingMessage.apply(tryToInsert.apply(sj -> sj.withRepeatSeconds(null).withCronExpression(BAD_CRON).withCronTimeZoneId("UTC"))) + .contains("Support for specifying both a day-of-week AND a day-of-month parameter is not implemented"); + + /////////////////// + // success cases // + /////////////////// + assertNoErrors.accept(tryToInsert.apply(sj -> sj.withCronExpression(GOOD_CRON).withCronTimeZoneId("UTC"))); + assertNoErrors.accept(tryToInsert.apply(sj -> sj.withRepeatSeconds(1))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostInsertActionSchedulesJob() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of( + new ScheduledJobParameter().withKey("processName").withValue(TestUtils.PROCESS_NAME_BASEPULL))))); + + assertTrue(CollectionUtils.nullSafeIsEmpty(insertOutput.getRecords().get(0).getErrors())); + assertTrue(CollectionUtils.nullSafeIsEmpty(insertOutput.getRecords().get(0).getWarnings())); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostInsertActionIssuesWarnings() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of( + new ScheduledJobParameter().withKey("processName").withValue("notAProcess"))))); + + assertTrue(CollectionUtils.nullSafeIsEmpty(insertOutput.getRecords().get(0).getErrors())); + assertThat(insertOutput.getRecords().get(0).getWarnings()) + .hasSize(1).first().extracting("message").asString() + .contains("Error scheduling job: Unrecognized processName"); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPreUpdateAssertValidationErrors() throws QException + { + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1))); + + UnsafeFunction, QRecord, QException> tryToUpdate = consumer -> + { + QRecord record = new QRecord().withValue("id", 1); + consumer.accept(record); + UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(ScheduledJob.TABLE_NAME).withRecord(record)); + return (updateOutput.getRecords().get(0)); + }; + + ///////////////////////////////////////////////////////// + // lambdas to run a test and assert about no of errors // + ///////////////////////////////////////////////////////// + Function> assertOneErrorExtractingMessage = qRecord -> assertThat(qRecord.getErrors()).hasSize(1).first().extracting("message").asString(); + Consumer assertNoErrors = qRecord -> assertThat(qRecord.getErrors()).hasSize(0); + + assertOneErrorExtractingMessage.apply(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null))) + .contains("Either Cron Expression or Repeat Seconds must be given"); + + assertOneErrorExtractingMessage.apply(tryToUpdate.apply(r -> r.withValue("cronExpression", GOOD_CRON).withValue("cronTimeZoneId", "UTC"))) + .contains("Cron Expression and Repeat Seconds may not both be given"); + + assertOneErrorExtractingMessage.apply(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", GOOD_CRON))) + .contains("If a Cron Expression is given, then a Cron Time Zone is required"); + + assertOneErrorExtractingMessage.apply(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", BAD_CRON).withValue("cronTimeZoneId", "UTC"))) + .contains("Support for specifying both a day-of-week AND a day-of-month parameter is not implemented"); + + /////////////////// + // success cases // + /////////////////// + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("modifyDate", Instant.now()))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", GOOD_CRON).withValue("cronTimeZoneId", "UTC"))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", GOOD_CRON_2).withValue("cronTimeZoneId", "UTC"))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", GOOD_CRON).withValue("cronTimeZoneId", "America/Chicago"))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", 1).withValue("cronExpression", null).withValue("cronTimeZoneId", null))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", 2))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostUpdateActionReSchedulesJob() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + ////////////////////////////////////////////////// + // do an insert - this will originally schedule // + ////////////////////////////////////////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of(new ScheduledJobParameter().withKey("processName").withValue(TestUtils.PROCESS_NAME_BASEPULL))))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + assertThat(wrappers.get(0).trigger()).isInstanceOf(SimpleTrigger.class); + + ////////////////////////////////////// + // now do an update, to re-schedule // + ////////////////////////////////////// + new UpdateAction().execute(new UpdateInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withId(1) + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(null) + .withCronExpression(GOOD_CRON) + .withCronTimeZoneId("UTC"))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + assertThat(wrappers.get(0).trigger()).isInstanceOf(CronTrigger.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostUpdateActionIssuesWarnings() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + ////////////////////////////////////////////////// + // do an insert - this will originally schedule // + ////////////////////////////////////////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of(new ScheduledJobParameter().withKey("processName").withValue(TestUtils.PROCESS_NAME_BASEPULL))))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + assertThat(wrappers.get(0).trigger()).isInstanceOf(SimpleTrigger.class); + + ////////////////////////////////////// + // now do an update, to re-schedule // + ////////////////////////////////////// + UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withId(1) + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(null) + .withCronExpression(GOOD_CRON) + .withCronTimeZoneId("UTC") + .withJobParameters(List.of(new ScheduledJobParameter().withKey("process").withValue("not"))))); + + assertTrue(CollectionUtils.nullSafeIsEmpty(updateOutput.getRecords().get(0).getErrors())); + assertThat(updateOutput.getRecords().get(0).getWarnings()) + .hasSize(1).first().extracting("message").asString() + .contains("Missing scheduledJobParameter with key [processName]"); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostDeleteUnschedules() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + ////////////////////////////////////////////////// + // do an insert - this will originally schedule // + ////////////////////////////////////////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of(new ScheduledJobParameter().withKey("processName").withValue(TestUtils.PROCESS_NAME_BASEPULL))))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + + //////////////////////////////////// + // now do a delete, to unschedule // + //////////////////////////////////// + new DeleteAction().execute(new DeleteInput(ScheduledJob.TABLE_NAME).withPrimaryKeys(List.of(1))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java new file mode 100644 index 00000000..68a7c37b --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java @@ -0,0 +1,207 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.util.ArrayList; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobParameter; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Unit test for QScheduleManager + *******************************************************************************/ +class QScheduleManagerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QLogger.deactivateCollectingLoggerForClass(QuartzScheduler.class); + + try + { + QScheduleManager.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + + try + { + QuartzScheduler.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private ScheduledJob newScheduledJob(ScheduledJobType type, Map params) + { + ScheduledJob scheduledJob = new ScheduledJob() + .withId(1) + .withIsActive(true) + .withSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME) + .withType(type.name()) + .withRepeatSeconds(1) + .withJobParameters(new ArrayList<>()); + + for(Map.Entry entry : params.entrySet()) + { + scheduledJob.getJobParameters().add(new ScheduledJobParameter().withKey(entry.getKey()).withValue(entry.getValue())); + } + + return (scheduledJob); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSetupScheduledJobErrorCases() throws QException + { + QScheduleManager qScheduleManager = QScheduleManager.initInstance(QContext.getQInstance(), () -> QContext.getQSession()); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, Map.of()).withRepeatSeconds(null))) + .hasMessageContaining("Missing a schedule"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, Map.of()).withType(null))) + .hasMessageContaining("Missing a type"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, Map.of()).withType("notAType"))) + .hasMessageContaining("Unrecognized type"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, Map.of()))) + .hasMessageContaining("Missing scheduledJobParameter with key [processName]"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, Map.of("processName", "notAProcess")))) + .hasMessageContaining("Unrecognized processName"); + + QContext.getQInstance().getProcess(TestUtils.PROCESS_NAME_BASEPULL).withSchedule(new QScheduleMetaData()); + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, Map.of("processName", TestUtils.PROCESS_NAME_BASEPULL)))) + .hasMessageContaining("has a schedule in its metaData - so it should not be dynamically scheduled"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.QUEUE_PROCESSOR, Map.of()))) + .hasMessageContaining("Missing scheduledJobParameter with key [queueName]"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.QUEUE_PROCESSOR, Map.of("queueName", "notAQueue")))) + .hasMessageContaining("Unrecognized queueName"); + + QContext.getQInstance().getQueue(TestUtils.TEST_SQS_QUEUE).withSchedule(new QScheduleMetaData()); + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.QUEUE_PROCESSOR, Map.of("queueName", TestUtils.TEST_SQS_QUEUE)))) + .hasMessageContaining("has a schedule in its metaData - so it should not be dynamically scheduled"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of()))) + .hasMessageContaining("Missing scheduledJobParameter with key [tableName]"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", "notATable")))) + .hasMessageContaining("Missing scheduledJobParameter with key [automationStatus]"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", "notATable", "automationStatus", AutomationStatus.PENDING_INSERT_AUTOMATIONS.name())))) + .hasMessageContaining("Unrecognized tableName"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC, "automationStatus", AutomationStatus.PENDING_INSERT_AUTOMATIONS.name())))) + .hasMessageContaining("does not have automationDetails"); + + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().withSchedule(null); + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY, "automationStatus", "foobar")))) + .hasMessageContaining("Did not find table automation actions matching automationStatus") + .hasMessageContaining("Found: PENDING_INSERT_AUTOMATIONS,PENDING_UPDATE_AUTOMATIONS"); + + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().withSchedule(new QScheduleMetaData()); + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY, "automationStatus", AutomationStatus.PENDING_INSERT_AUTOMATIONS.name())))) + .hasMessageContaining("has a schedule in its metaData - so it should not be dynamically scheduled"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessfulScheduleWithQuartz() throws QException + { + QCollectingLogger quartzLogger = QLogger.activateCollectingLoggerForClass(QuartzScheduler.class); + + QInstance qInstance = QContext.getQInstance(); + QuartzTestUtils.setupInstanceForQuartzTests(); + + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> QContext.getQSession()); + qScheduleManager.start(); + + qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, + Map.of("processName", TestUtils.PROCESS_NAME_GREET_PEOPLE)) + .withId(2) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + + qInstance.getQueue(TestUtils.TEST_SQS_QUEUE).setSchedule(null); + qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.QUEUE_PROCESSOR, + Map.of("queueName", TestUtils.TEST_SQS_QUEUE)) + .withId(3) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().setSchedule(null); + qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, + Map.of("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY, "automationStatus", AutomationStatus.PENDING_UPDATE_AUTOMATIONS.name())) + .withId(4) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + + assertThat(quartzLogger.getCollectedMessages()) + .anyMatch(l -> l.getMessage().matches(".*Scheduled new job.*PROCESS.scheduledJob:2.*")) + .anyMatch(l -> l.getMessage().matches(".*Scheduled new job.*QUEUE_PROCESSOR.scheduledJob:3.*")) + .anyMatch(l -> l.getMessage().matches(".*Scheduled new job.*TABLE_AUTOMATIONS.scheduledJob:4.*")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java new file mode 100644 index 00000000..16b1a427 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java @@ -0,0 +1,79 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SchedulerTestUtils +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessMetaData buildTestProcess(String name, String schedulerName) + { + return new QProcessMetaData() + .withName(name) + .withSchedule(new QScheduleMetaData() + .withSchedulerName(schedulerName) + .withRepeatMillis(2) + .withInitialDelaySeconds(0)) + .withStepList(List.of(new QBackendStepMetaData() + .withName("step") + .withCode(new QCodeReference(BasicStep.class)))); + } + + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class BasicStep implements BackendStep + { + public static int counter = 0; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + counter++; + } + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java new file mode 100644 index 00000000..46f5c8cc --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java @@ -0,0 +1,167 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils.BasicStep; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableSQSQueueRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableTableAutomationsRunner; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.logging.log4j.Level; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.quartz.SchedulerException; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for QuartzScheduler + *******************************************************************************/ +class QuartzSchedulerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QuartzTestUtils.afterEach(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws Exception + { + try + { + QInstance qInstance = QContext.getQInstance(); + QuartzTestUtils.setupInstanceForQuartzTests(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // set these runners to use collecting logger, so we can assert that they did run, and didn't throw // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + QCollectingLogger quartzSqsPollerJobLog = QLogger.activateCollectingLoggerForClass(SchedulableSQSQueueRunner.class); + QCollectingLogger quartzTableAutomationsJobLog = QLogger.activateCollectingLoggerForClass(SchedulableTableAutomationsRunner.class); + + ////////////////////////////////////////// + // add a process we can run and observe // + ////////////////////////////////////////// + qInstance.addProcess(SchedulerTestUtils.buildTestProcess("testScheduledProcess", QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + + ///////////////////////////////////////////////////////////////////// + // start the schedule manager, then ask it to set up all schedules // + ///////////////////////////////////////////////////////////////////// + QSession qSession = QContext.getQSession(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); + qScheduleManager.start(); + qScheduleManager.setupAllSchedules(); + + ////////////////////////////////////////////////// + // give a moment for the job to run a few times // + ////////////////////////////////////////////////// + SleepUtils.sleep(150, TimeUnit.MILLISECONDS); + qScheduleManager.stopAsync(); + + System.out.println("Ran: " + BasicStep.counter + " times"); + assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice (but only ran [" + BasicStep.counter + "] time(s))."); + + ////////////////////////////////////////////////////// + // make sure poller ran, and didn't issue any warns // + ////////////////////////////////////////////////////// + assertThat(quartzSqsPollerJobLog.getCollectedMessages()) + .anyMatch(m -> m.getLevel().equals(Level.DEBUG) && m.getMessage().contains("Running SQS Queue poller")) + .noneMatch(m -> m.getLevel().equals(Level.WARN)); + + ////////////////////////////////////////////////////// + // make sure poller ran, and didn't issue any warns // + ////////////////////////////////////////////////////// + assertThat(quartzTableAutomationsJobLog.getCollectedMessages()) + .anyMatch(m -> m.getLevel().equals(Level.DEBUG) && m.getMessage().contains("Running Table Automations")) + .noneMatch(m -> m.getLevel().equals(Level.WARN)); + } + finally + { + QLogger.deactivateCollectingLoggerForClass(SchedulableSQSQueueRunner.class); + QLogger.deactivateCollectingLoggerForClass(SchedulableTableAutomationsRunner.class); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRemovingNoLongerNeededJobsDuringSetupSchedules() throws SchedulerException + { + QInstance qInstance = QContext.getQInstance(); + QuartzTestUtils.setupInstanceForQuartzTests(); + + //////////////////////////// + // put two jobs in quartz // + //////////////////////////// + QProcessMetaData test1 = SchedulerTestUtils.buildTestProcess("test1", QuartzTestUtils.QUARTZ_SCHEDULER_NAME); + QProcessMetaData test2 = SchedulerTestUtils.buildTestProcess("test2", QuartzTestUtils.QUARTZ_SCHEDULER_NAME); + qInstance.addProcess(test1); + qInstance.addProcess(test2); + + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.PROCESS.name()); + + QuartzScheduler quartzScheduler = QuartzScheduler.initInstance(qInstance, QuartzTestUtils.QUARTZ_SCHEDULER_NAME, QuartzTestUtils.getQuartzProperties(), () -> QContext.getQSession()); + quartzScheduler.start(); + + quartzScheduler.setupSchedulable(new BasicSchedulableIdentity("process:test1", null), schedulableType, Collections.emptyMap(), test1.getSchedule(), false); + quartzScheduler.setupSchedulable(new BasicSchedulableIdentity("process:test2", null), schedulableType, Collections.emptyMap(), test1.getSchedule(), false); + + quartzScheduler.startOfSetupSchedules(); + quartzScheduler.setupSchedulable(new BasicSchedulableIdentity("process:test1", null), schedulableType, Collections.emptyMap(), test1.getSchedule(), false); + quartzScheduler.endOfSetupSchedules(); + + List quartzJobAndTriggerWrappers = quartzScheduler.queryQuartz(); + assertEquals(1, quartzJobAndTriggerWrappers.size()); + assertEquals("process:test1", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()); + } + + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java new file mode 100644 index 00000000..a82d163c --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java @@ -0,0 +1,137 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import java.util.List; +import java.util.Properties; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import org.quartz.SchedulerException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuartzTestUtils +{ + public final static String QUARTZ_SCHEDULER_NAME = "TestQuartzScheduler"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Properties getQuartzProperties() + { + Properties quartzProperties = new Properties(); + quartzProperties.put("org.quartz.scheduler.instanceName", QUARTZ_SCHEDULER_NAME); + quartzProperties.put("org.quartz.threadPool.threadCount", "3"); + quartzProperties.put("org.quartz.jobStore.class", "org.quartz.simpl.RAMJobStore"); + return (quartzProperties); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void setupInstanceForQuartzTests() + { + QInstance qInstance = QContext.getQInstance(); + + /////////////////////////////////////////////////// + // remove the simple scheduler from the instance // + /////////////////////////////////////////////////// + qInstance.getSchedulers().clear(); + + //////////////////////////////////////////////////////// + // add the quartz scheduler meta-data to the instance // + //////////////////////////////////////////////////////// + qInstance.addScheduler(new QuartzSchedulerMetaData() + .withProperties(getQuartzProperties()) + .withName(QUARTZ_SCHEDULER_NAME)); + + /////////////////////////////////////////////////////////////////////////////////// + // set the queue providers & automation providers to use the quartz scheduler // + // also, set their initial delay to avoid default delay done by our scheduler // + // (that gives us a chance to re-pause if re-scheduling a previously paused job) // + /////////////////////////////////////////////////////////////////////////////////// + qInstance.getTables().values().forEach(t -> + { + if(t.getAutomationDetails() != null) + { + t.getAutomationDetails().getSchedule() + .withSchedulerName(QUARTZ_SCHEDULER_NAME) + .withInitialDelayMillis(1); + } + }); + + qInstance.getQueues().values() + .forEach(q -> q.getSchedule() + .withSchedulerName(QUARTZ_SCHEDULER_NAME) + .withInitialDelayMillis(1)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static List queryQuartz() throws SchedulerException + { + return QuartzScheduler.getInstance().queryQuartz(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void afterEach() + { + try + { + QScheduleManager.getInstance().stop(); + QScheduleManager.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + + try + { + QuartzScheduler.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java new file mode 100644 index 00000000..f2bf6067 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java @@ -0,0 +1,285 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import java.util.List; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.quartz.SchedulerException; +import org.quartz.Trigger; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit tests for the various quartz management processes + *******************************************************************************/ +class QuartzJobsProcessTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws QException + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addTable(new QTableMetaData() + .withName("quartzJobDetails") + .withBackendName(TestUtils.MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.LONG))); + MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, QuartzScheduler.class.getPackageName()); + + QuartzTestUtils.setupInstanceForQuartzTests(); + + ////////////////////////////////////////////////////////////////////////////// + // start the schedule manager, which will schedule things, and start quartz // + ////////////////////////////////////////////////////////////////////////////// + QSession qSession = QContext.getQSession(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); + qScheduleManager.start(); + qScheduleManager.setupAllSchedules(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QuartzTestUtils.afterEach(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPauseAllQuartzJobs() throws QException, SchedulerException + { + //////////////////////////////////////// + // make sure nothing starts as paused // + //////////////////////////////////////// + assertNoneArePaused(); + + /////////////////////////////// + // run the pause-all process // + /////////////////////////////// + RunProcessInput input = new RunProcessInput(); + input.setProcessName(PauseAllQuartzJobsProcess.class.getSimpleName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + ////////////////////////////////////// + // assert everything becomes paused // + ////////////////////////////////////// + assertAllArePaused(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testResumeAllQuartzJobs() throws QException, SchedulerException + { + /////////////////////////////// + // run the pause-all process // + /////////////////////////////// + RunProcessInput input = new RunProcessInput(); + input.setProcessName(PauseAllQuartzJobsProcess.class.getSimpleName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + SleepUtils.sleep(3, TimeUnit.SECONDS); + + ////////////////////////////////////// + // assert everything becomes paused // + ////////////////////////////////////// + assertAllArePaused(); + + //////////////////// + // run resume all // + //////////////////// + input = new RunProcessInput(); + input.setProcessName(ResumeAllQuartzJobsProcess.class.getSimpleName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + //////////////////////////////////////// + // make sure nothing ends up as paused // + //////////////////////////////////////// + assertNoneArePaused(); + + //////////////////// + // pause just one // + //////////////////// + List quartzJobAndTriggerWrappers = QuartzTestUtils.queryQuartz(); + new InsertAction().execute(new InsertInput("quartzJobDetails").withRecord(new QRecord() + .withValue("jobName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()) + .withValue("jobGroup", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getGroup()) + )); + + input = new RunProcessInput(); + input.setProcessName(PauseQuartzJobsProcess.class.getSimpleName()); + input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter())); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + ///////////////////////////////////////////////////////// + // make sure at least 1 is paused, some are not paused // + ///////////////////////////////////////////////////////// + assertAnyAre(Trigger.TriggerState.PAUSED); + assertAnyAreNot(Trigger.TriggerState.PAUSED); + + ////////////////////////// + // run resume all again // + ////////////////////////// + input = new RunProcessInput(); + input.setProcessName(ResumeAllQuartzJobsProcess.class.getSimpleName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + //////////////////////////////////////// + // make sure nothing ends up as paused // + //////////////////////////////////////// + assertNoneArePaused(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPauseOneResumeOne() throws QException, SchedulerException + { + ///////////////////////////////////// + // make sure nothing starts paused // + ///////////////////////////////////// + assertNoneArePaused(); + + //////////////////// + // pause just one // + //////////////////// + List quartzJobAndTriggerWrappers = QuartzTestUtils.queryQuartz(); + new InsertAction().execute(new InsertInput("quartzJobDetails").withRecord(new QRecord() + .withValue("jobName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()) + .withValue("jobGroup", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getGroup()) + )); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName(PauseQuartzJobsProcess.class.getSimpleName()); + input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter())); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + ///////////////////////////////////////////////////////// + // make sure at least 1 is paused, some are not paused // + ///////////////////////////////////////////////////////// + assertAnyAre(Trigger.TriggerState.PAUSED); + assertAnyAreNot(Trigger.TriggerState.PAUSED); + + ///////////////////////////////////////////////////////////////////////////// + // now resume the same one (will still be only row in our in-memory table) // + ///////////////////////////////////////////////////////////////////////////// + input = new RunProcessInput(); + input.setProcessName(ResumeQuartzJobsProcess.class.getSimpleName()); + input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter())); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + ////////////////////////////////////// + // make sure nothing ends up paused // + ////////////////////////////////////// + assertNoneArePaused(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void assertAnyAre(Trigger.TriggerState triggerState) throws SchedulerException + { + assertThat(QuartzTestUtils.queryQuartz()).anyMatch(qjtw -> qjtw.triggerState().equals(triggerState)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void assertAnyAreNot(Trigger.TriggerState triggerState) throws SchedulerException + { + assertThat(QuartzTestUtils.queryQuartz()).anyMatch(qjtw -> !qjtw.triggerState().equals(triggerState)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void assertNoneArePaused() throws SchedulerException + { + assertThat(QuartzTestUtils.queryQuartz()).noneMatch(qjtw -> qjtw.triggerState().equals(Trigger.TriggerState.PAUSED)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void assertAllArePaused() throws SchedulerException + { + assertThat(QuartzTestUtils.queryQuartz()).allMatch(qjtw -> qjtw.triggerState().equals(Trigger.TriggerState.PAUSED)); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java similarity index 55% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java index b164376f..827cc57e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,36 +19,32 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.scheduler; +package com.kingsrook.qqq.backend.core.scheduler.simple; -import java.util.List; import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils.BasicStep; import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* ** Unit test for ScheduleManager *******************************************************************************/ -class ScheduleManagerTest extends BaseTest +class SimpleSchedulerTest extends BaseTest { /******************************************************************************* @@ -57,7 +53,7 @@ class ScheduleManagerTest extends BaseTest @AfterEach void afterEach() { - ScheduleManager.resetSingleton(); + QScheduleManager.getInstance().unInit(); } @@ -66,15 +62,18 @@ class ScheduleManagerTest extends BaseTest ** *******************************************************************************/ @Test - void testStartAndStop() + void testStartAndStop() throws QException { - QInstance qInstance = QContext.getQInstance(); - ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance); - scheduleManager.start(); + QInstance qInstance = QContext.getQInstance(); - assertThat(scheduleManager.getExecutors()).isNotEmpty(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> QContext.getQSession()); + qScheduleManager.start(); - scheduleManager.stopAsync(); + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); + assertThat(simpleScheduler.getExecutors()).isNotEmpty(); + + qScheduleManager.stop(); + simpleScheduler.getExecutors().forEach(e -> assertEquals(StandardScheduledExecutor.RunningState.STOPPED, e.getRunningState())); } @@ -83,50 +82,30 @@ class ScheduleManagerTest extends BaseTest ** *******************************************************************************/ @Test - void testScheduledProcess() throws QInstanceValidationException + void testScheduledProcess() throws QException { QInstance qInstance = QContext.getQInstance(); new QInstanceValidator().validate(qInstance); qInstance.getAutomationProviders().clear(); qInstance.getQueueProviders().clear(); - qInstance.addProcess( - new QProcessMetaData() - .withName("testScheduledProcess") - .withSchedule(new QScheduleMetaData().withRepeatMillis(2).withInitialDelaySeconds(0)) - .withStepList(List.of(new QBackendStepMetaData() - .withName("step") - .withCode(new QCodeReference(BasicStep.class)))) - ); + qInstance.addProcess(SchedulerTestUtils.buildTestProcess("testScheduledProcess", TestUtils.SIMPLE_SCHEDULER_NAME)); BasicStep.counter = 0; - ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance); - scheduleManager.setSessionSupplier(QSession::new); - scheduleManager.start(); + QSession qSession = QContext.getQSession(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); + qScheduleManager.start(); + + ////////////////////////////////////////////////// + // give a moment for the job to run a few times // + ////////////////////////////////////////////////// SleepUtils.sleep(50, TimeUnit.MILLISECONDS); - scheduleManager.stopAsync(); + qScheduleManager.stopAsync(); System.out.println("Ran: " + BasicStep.counter + " times"); - assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice"); + assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice (but only ran [" + BasicStep.counter + "] time(s)."); } - - /******************************************************************************* - ** - *******************************************************************************/ - public static class BasicStep implements BackendStep - { - static int counter = 0; - - - - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - counter++; - } - } - } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 2e3a937e..0086c351 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -92,6 +92,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple.SimpleSchedulerMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; @@ -176,6 +179,9 @@ public class TestUtils public static final String SECURITY_KEY_TYPE_STORE_NULL_BEHAVIOR = "storeNullBehavior"; public static final String SECURITY_KEY_TYPE_INTERNAL_OR_EXTERNAL = "internalOrExternal"; + public static final String SIMPLE_SCHEDULER_NAME = "simpleScheduler"; + public static final String TEST_SQS_QUEUE = "testSQSQueue"; + /******************************************************************************* @@ -237,11 +243,23 @@ public class TestUtils defineWidgets(qInstance); defineApps(qInstance); + qInstance.addScheduler(defineSimpleScheduler()); + return (qInstance); } + /******************************************************************************* + ** + *******************************************************************************/ + private static QSchedulerMetaData defineSimpleScheduler() + { + return new SimpleSchedulerMetaData().withName(SIMPLE_SCHEDULER_NAME); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -726,6 +744,9 @@ public class TestUtils { return (new QTableAutomationDetails() .withProviderName(POLLING_AUTOMATION) + .withSchedule(new QScheduleMetaData() + .withSchedulerName(SIMPLE_SCHEDULER_NAME) + .withRepeatSeconds(60)) .withStatusTracking(new AutomationStatusTracking() .withType(AutomationStatusTrackingType.FIELD_IN_TABLE) .withFieldName("qqqAutomationStatus"))); @@ -1324,10 +1345,13 @@ public class TestUtils private static QQueueMetaData defineTestSqsQueue() { return (new QQueueMetaData() - .withName("testSQSQueue") + .withName(TEST_SQS_QUEUE) .withProviderName(DEFAULT_QUEUE_PROVIDER) .withQueueName("test-queue") - .withProcessName(PROCESS_NAME_INCREASE_BIRTHDATE)); + .withProcessName(PROCESS_NAME_INCREASE_BIRTHDATE) + .withSchedule(new QScheduleMetaData() + .withRepeatSeconds(60) + .withSchedulerName(SIMPLE_SCHEDULER_NAME))); } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index df54eef9..ea07888a 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -43,6 +43,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -368,13 +369,19 @@ public abstract class AbstractBaseFilesystemAction { try { - Optional tableCustomizer = QCodeLoader.getTableCustomizer(AbstractPostReadFileCustomizer.class, table, FilesystemTableCustomizers.POST_READ_FILE.getRole()); - if(tableCustomizer.isEmpty()) + Optional codeReference = table.getCustomizer(FilesystemTableCustomizers.POST_READ_FILE.getRole()); + if(codeReference.isEmpty()) { return (fileContents); } - return tableCustomizer.get().customizeFileContents(fileContents); + AbstractPostReadFileCustomizer tableCustomizer = QCodeLoader.getAdHoc(AbstractPostReadFileCustomizer.class, codeReference.get()); + if(tableCustomizer == null) + { + return (fileContents); + } + + return tableCustomizer.customizeFileContents(fileContents); } catch(Exception e) { diff --git a/qqq-backend-module-mongodb/pom.xml b/qqq-backend-module-mongodb/pom.xml index 58ddb23b..4aa23c36 100644 --- a/qqq-backend-module-mongodb/pom.xml +++ b/qqq-backend-module-mongodb/pom.xml @@ -50,11 +50,6 @@ mongodb-driver-sync 4.11.1 - - org.apache.logging.log4j - log4j-slf4j-impl - 2.23.0 - org.testcontainers diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index 5ba71e75..fccc54c7 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -153,7 +153,7 @@ public abstract class AbstractRDBMSAction if("".equals(value)) { QFieldType type = field.getType(); - if(type.equals(QFieldType.INTEGER) || type.equals(QFieldType.DECIMAL) || type.equals(QFieldType.DATE) || type.equals(QFieldType.DATE_TIME) || type.equals(QFieldType.BOOLEAN)) + if(type.equals(QFieldType.INTEGER) || type.equals(QFieldType.LONG) || type.equals(QFieldType.DECIMAL) || type.equals(QFieldType.DATE) || type.equals(QFieldType.DATE_TIME) || type.equals(QFieldType.BOOLEAN)) { value = null; } @@ -853,6 +853,10 @@ public abstract class AbstractRDBMSAction { return (QueryManager.getInteger(resultSet, i)); } + case LONG: + { + return (QueryManager.getLong(resultSet, i)); + } case DECIMAL: { return (QueryManager.getBigDecimal(resultSet, i)); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index ce720120..4e0d0fad 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -143,7 +143,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega QFieldType fieldType = aggregate.getFieldType(); if(fieldType == null) { - if(field.getType().equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG))) + if((field.getType().equals(QFieldType.INTEGER) || field.getType().equals(QFieldType.LONG)) && (aggregate.getOperator().equals(AggregateOperator.AVG))) { fieldType = QFieldType.DECIMAL; } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java index 6714979b..27d5adba 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java @@ -40,26 +40,51 @@ public class ConnectionManager *******************************************************************************/ public Connection getConnection(RDBMSBackendMetaData backend) throws SQLException { - String jdbcURL; - - if(StringUtils.hasContent(backend.getJdbcUrl())) - { - jdbcURL = backend.getJdbcUrl(); - } - else - { - switch(backend.getVendor()) - { - // TODO aws-mysql-jdbc driver not working when running on AWS - // jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=CONVERT_TO_NULL"; - case "aurora" -> jdbcURL = "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false"; - case "mysql" -> jdbcURL = "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull"; - case "h2" -> jdbcURL = "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1"; - default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor()); - } - } - + String jdbcURL = getJdbcUrl(backend); return DriverManager.getConnection(jdbcURL, backend.getUsername(), backend.getPassword()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getJdbcDriverClassName(RDBMSBackendMetaData backend) + { + if(StringUtils.hasContent(backend.getJdbcDriverClassName())) + { + return backend.getJdbcDriverClassName(); + } + + return switch(backend.getVendor()) + { + case "mysql", "aurora" -> "com.mysql.cj.jdbc.Driver"; + case "h2" -> "org.h2.Driver"; + default -> throw (new IllegalStateException("We do not know what jdbc driver to use for vendor name [" + backend.getVendor() + "]. Try setting jdbcDriverClassName in your backend meta data.")); + }; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getJdbcUrl(RDBMSBackendMetaData backend) + { + if(StringUtils.hasContent(backend.getJdbcUrl())) + { + return backend.getJdbcUrl(); + } + + return switch(backend.getVendor()) + { + // TODO aws-mysql-jdbc driver not working when running on AWS + // jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=CONVERT_TO_NULL"; + case "aurora" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false"; + case "mysql" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull"; + case "h2" -> "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1"; + default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor()); + }; + } + } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java index 6ecc6e8c..a86a6f45 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java @@ -40,6 +40,7 @@ public class RDBMSBackendMetaData extends QBackendMetaData private String password; private String jdbcUrl; + private String jdbcDriverClassName; @@ -314,4 +315,35 @@ public class RDBMSBackendMetaData extends QBackendMetaData return (this); } + + /******************************************************************************* + ** Getter for jdbcDriverClassName + *******************************************************************************/ + public String getJdbcDriverClassName() + { + return (this.jdbcDriverClassName); + } + + + + /******************************************************************************* + ** Setter for jdbcDriverClassName + *******************************************************************************/ + public void setJdbcDriverClassName(String jdbcDriverClassName) + { + this.jdbcDriverClassName = jdbcDriverClassName; + } + + + + /******************************************************************************* + ** Fluent setter for jdbcDriverClassName + *******************************************************************************/ + public RDBMSBackendMetaData withJdbcDriverClassName(String jdbcDriverClassName) + { + this.jdbcDriverClassName = jdbcDriverClassName; + return (this); + } + + } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java new file mode 100644 index 00000000..7fc556d4 --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java @@ -0,0 +1,209 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.model.metadata; + + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; + + +/******************************************************************************* + ** note - this class is pretty mysql-specific + *******************************************************************************/ +public class RDBMSTableMetaDataBuilder +{ + private static final QLogger LOG = QLogger.getLogger(RDBMSTableMetaDataBuilder.class); + + private boolean useCamelCaseNames = true; + + private static Map typeMap = new HashMap<>(); + + static + { + //////////////////////////////////////////////// + // these are types as returned by mysql // + // let null in here mean unsupported QQQ type // + //////////////////////////////////////////////// + typeMap.put("TEXT", QFieldType.TEXT); + typeMap.put("BINARY", QFieldType.BLOB); + typeMap.put("SET", null); + typeMap.put("VARBINARY", QFieldType.BLOB); + typeMap.put("MEDIUMBLOB", QFieldType.BLOB); + typeMap.put("NUMERIC", QFieldType.INTEGER); + typeMap.put("INTEGER", QFieldType.INTEGER); + typeMap.put("BIGINT UNSIGNED", QFieldType.LONG); + typeMap.put("MEDIUMINT UNSIGNED", QFieldType.INTEGER); + typeMap.put("SMALLINT UNSIGNED", QFieldType.INTEGER); + typeMap.put("TINYINT UNSIGNED", QFieldType.INTEGER); + typeMap.put("BIT", null); + typeMap.put("FLOAT", null); + typeMap.put("REAL", null); + typeMap.put("VARCHAR", QFieldType.STRING); + typeMap.put("BOOL", QFieldType.BOOLEAN); + typeMap.put("YEAR", null); + typeMap.put("TIME", QFieldType.TIME); + typeMap.put("TIMESTAMP", QFieldType.DATE_TIME); + } + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData buildTableMetaData(RDBMSBackendMetaData backendMetaData, String tableName) throws QException + { + try(Connection connection = new ConnectionManager().getConnection(backendMetaData)) + { + List fieldMetaDataList = new ArrayList<>(); + String primaryKey = null; + + DatabaseMetaData databaseMetaData = connection.getMetaData(); + Map dataTypeMap = new HashMap<>(); + ResultSet typeInfoResultSet = databaseMetaData.getTypeInfo(); + while(typeInfoResultSet.next()) + { + String name = typeInfoResultSet.getString("TYPE_NAME"); + Integer id = typeInfoResultSet.getInt("DATA_TYPE"); + dataTypeMap.put(id, name); + } + + // todo - for h2, uppercase both db & table names... + String databaseName = backendMetaData.getDatabaseName(); // these work for mysql - unclear about other vendors. + String schemaName = null; + String tableNameForMetaDataQueries = tableName; + + if(backendMetaData.getVendor().equals("h2")) + { + databaseName = databaseName.toUpperCase(); + tableNameForMetaDataQueries = tableName.toUpperCase(); + } + + try(ResultSet tableResultSet = databaseMetaData.getTables(databaseName, schemaName, tableNameForMetaDataQueries, null)) + { + if(!tableResultSet.next()) + { + throw (new QException("Table: " + tableName + " was not found in backend: " + backendMetaData.getName())); + } + } + + try(ResultSet columnsResultSet = databaseMetaData.getColumns(databaseName, schemaName, tableNameForMetaDataQueries, null)) + { + while(columnsResultSet.next()) + { + String columnName = columnsResultSet.getString("COLUMN_NAME"); + String columnSize = columnsResultSet.getString("COLUMN_SIZE"); + Integer dataTypeId = columnsResultSet.getInt("DATA_TYPE"); + String isNullable = columnsResultSet.getString("IS_NULLABLE"); + String isAutoIncrement = columnsResultSet.getString("IS_AUTOINCREMENT"); + + String dataTypeName = dataTypeMap.get(dataTypeId); + QFieldType type = typeMap.get(dataTypeName); + if(type == null) + { + LOG.info("Table " + tableName + " column " + columnName + " has an unmapped type: " + dataTypeId + ". Field will not be added to QTableMetaData"); + continue; + } + + String qqqFieldName = QInstanceEnricher.inferNameFromBackendName(columnName); + + QFieldMetaData fieldMetaData = new QFieldMetaData(qqqFieldName, type) + // todo - what string? .withIsRequired(!isNullable) + .withBackendName(columnName); + + fieldMetaDataList.add(fieldMetaData); + + if("YES".equals(isAutoIncrement)) + { + primaryKey = qqqFieldName; + } + } + } + + if(fieldMetaDataList.isEmpty()) + { + throw (new QException("Could not find any usable fields in table: " + tableName)); + } + + if(primaryKey == null) + { + throw (new QException("Could not find primary key in table: " + tableName)); + } + + String qqqTableName = QInstanceEnricher.inferNameFromBackendName(tableName); + + QTableMetaData tableMetaData = new QTableMetaData() + .withBackendName(backendMetaData.getName()) + .withName(qqqTableName) + .withBackendDetails(new RDBMSTableBackendDetails().withTableName(tableName)) + .withFields(fieldMetaDataList) + .withPrimaryKeyField(primaryKey); + + return (tableMetaData); + } + catch(Exception e) + { + throw (new QException("Error automatically building table meta data for table: " + tableName, e)); + } + } + + + /******************************************************************************* + ** Getter for useCamelCaseNames + *******************************************************************************/ + public boolean getUseCamelCaseNames() + { + return (this.useCamelCaseNames); + } + + + + /******************************************************************************* + ** Setter for useCamelCaseNames + *******************************************************************************/ + public void setUseCamelCaseNames(boolean useCamelCaseNames) + { + this.useCamelCaseNames = useCamelCaseNames; + } + + + + /******************************************************************************* + ** Fluent setter for useCamelCaseNames + *******************************************************************************/ + public RDBMSTableMetaDataBuilder withUseCamelCaseNames(boolean useCamelCaseNames) + { + this.useCamelCaseNames = useCamelCaseNames; + return (this); + } + + +} diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilderTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilderTest.java new file mode 100644 index 00000000..c6b8fc17 --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilderTest.java @@ -0,0 +1,70 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.model.metadata; + + +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.rdbms.BaseTest; +import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for RDBMSTableMetaDataBuilder + *******************************************************************************/ +class RDBMSTableMetaDataBuilderTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + TestUtils.primeTestDatabase("prime-test-database.sql"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + QBackendMetaData backend = QContext.getQInstance().getBackend(TestUtils.DEFAULT_BACKEND_NAME); + QTableMetaData table = new RDBMSTableMetaDataBuilder().buildTableMetaData((RDBMSBackendMetaData) backend, "order"); + + assertNotNull(table); + assertEquals("order", table.getName()); + assertEquals("id", table.getPrimaryKeyField()); + assertEquals(QFieldType.INTEGER, table.getField(table.getPrimaryKeyField()).getType()); + assertNotNull(table.getField("storeId")); + } + +} \ No newline at end of file diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java index 93228b02..a4ad7378 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java @@ -1686,7 +1686,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction "string"; - case INTEGER -> "integer"; + case INTEGER, LONG -> "integer"; // todo - we could give 'format' w/ int32 & int64 to further specify case DECIMAL -> "number"; case BOOLEAN -> "boolean"; }; diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaDataContainer.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaDataContainer.java index 71fcac0a..2ea76133 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaDataContainer.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaDataContainer.java @@ -40,6 +40,17 @@ public class ApiInstanceMetaDataContainer extends QSupplementalInstanceMetaData + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getName() + { + return ApiSupplementType.NAME; + } + + + /******************************************************************************* ** Constructor ** diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 94ae0e71..6b674b09 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -710,10 +710,15 @@ public class QJavalinImplementation { throw (new QUserFacingException("Error updating " + tableMetaData.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getErrors()))); } - if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings())) - { - throw (new QUserFacingException("Warning updating " + tableMetaData.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getWarnings()))); - } + + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // at one time, we threw upon warning - but // + // on insert we need to return the record (e.g., to get a generated id), so, make update do the same. // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings())) + // { + // throw (new QUserFacingException("Warning updating " + tableMetaData.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getWarnings()))); + // } QJavalinAccessLogger.logEndSuccess(); context.result(JsonUtils.toJson(updateOutput)); @@ -779,9 +784,31 @@ public class QJavalinImplementation { String fieldName = formParam.getKey(); List values = formParam.getValue(); + if(CollectionUtils.nullSafeHasContents(values)) { String value = values.get(0); + + if("associations".equals(fieldName) && StringUtils.hasContent(value)) + { + JSONObject associationsJSON = new JSONObject(value); + for(String key : associationsJSON.keySet()) + { + JSONArray associatedRecords = associationsJSON.getJSONArray(key); + for(int i = 0; i < associatedRecords.length(); i++) + { + QRecord associatedRecord = new QRecord(); + JSONObject recordJSON = associatedRecords.getJSONObject(i); + for(String k : recordJSON.keySet()) + { + associatedRecord.withValue(k, ValueUtils.getValueAsString(recordJSON.get(k))); + } + record.withAssociatedRecord(key, associatedRecord); + } + } + continue; + } + if(StringUtils.hasContent(value)) { record.setValue(fieldName, value); @@ -793,7 +820,6 @@ public class QJavalinImplementation } else { - // is this ever hit? record.setValue(fieldName, null); } } @@ -881,10 +907,16 @@ public class QJavalinImplementation { throw (new QUserFacingException("Error inserting " + table.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getErrors()))); } - if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings())) - { - throw (new QUserFacingException("Warning inserting " + table.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getWarnings()))); - } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // at one time, we threw upon warning - but // + // our use-case is, the frontend, it wants to get the record, and show a success (with the generated id) // + // and then to also show a warning message - so - let it all be returned and handled on the frontend. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings())) + // { + // throw (new QUserFacingException("Warning inserting " + table.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getWarnings()))); + // } QJavalinAccessLogger.logEndSuccess(logPair("primaryKey", () -> (outputRecord.getValue(table.getPrimaryKeyField())))); context.result(JsonUtils.toJson(insertOutput)); diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java index a352e558..76a879b0 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java @@ -514,6 +514,42 @@ class QJavalinImplementationTest extends QJavalinTestBase + /******************************************************************************* + ** test an insert that returns a warning + ** + *******************************************************************************/ + @Test + public void test_dataInsertWithWarning() + { + Map body = new HashMap<>(); + body.put("firstName", "Warning"); + body.put("lastName", "Kelkhoff"); + body.put("email", "warning@kelkhoff.com"); + + HttpResponse response = Unirest.post(BASE_URL + "/data/person") + .header("Content-Type", "application/json") + .body(body) + .asString(); + + assertEquals(200, response.getStatus()); + JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody()); + assertTrue(jsonObject.has("records")); + JSONArray records = jsonObject.getJSONArray("records"); + assertEquals(1, records.length()); + JSONObject record0 = records.getJSONObject(0); + assertTrue(record0.has("values")); + JSONObject values0 = record0.getJSONObject("values"); + assertTrue(values0.has("id")); + assertEquals(7, values0.getInt("id")); + + assertTrue(record0.has("warnings")); + JSONArray warnings = record0.getJSONArray("warnings"); + assertEquals(1, warnings.length()); + assertTrue(warnings.getJSONObject(0).has("message")); + } + + + /******************************************************************************* ** test an insert - posting a multipart form. ** @@ -594,6 +630,52 @@ class QJavalinImplementationTest extends QJavalinTestBase + /******************************************************************************* + ** test an update - with a warning returned + ** + *******************************************************************************/ + @Test + public void test_dataUpdateWithWarning() + { + Map body = new HashMap<>(); + body.put("firstName", "Warning"); + body.put("birthDate", ""); + + HttpResponse response = Unirest.patch(BASE_URL + "/data/person/4") + .header("Content-Type", "application/json") + .body(body) + .asString(); + + assertEquals(200, response.getStatus()); + JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody()); + assertTrue(jsonObject.has("records")); + JSONArray records = jsonObject.getJSONArray("records"); + assertEquals(1, records.length()); + JSONObject record0 = records.getJSONObject(0); + assertTrue(record0.has("values")); + assertEquals("person", record0.getString("tableName")); + JSONObject values0 = record0.getJSONObject("values"); + assertEquals(4, values0.getInt("id")); + assertEquals("Warning", values0.getString("firstName")); + + assertTrue(record0.has("warnings")); + JSONArray warnings = record0.getJSONArray("warnings"); + assertEquals(1, warnings.length()); + assertTrue(warnings.getJSONObject(0).has("message")); + + /////////////////////////////////////////////////////////////////// + // re-GET the record, and validate that birthDate was nulled out // + /////////////////////////////////////////////////////////////////// + response = Unirest.get(BASE_URL + "/data/person/4").asString(); + assertEquals(200, response.getStatus()); + jsonObject = JsonUtils.toJSONObject(response.getBody()); + assertTrue(jsonObject.has("values")); + JSONObject values = jsonObject.getJSONObject("values"); + assertFalse(values.has("birthDate")); + } + + + /******************************************************************************* ** test an update - posting the data as a multipart form ** diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java index 430f72ae..ae5d2694 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java @@ -25,6 +25,10 @@ package com.kingsrook.qqq.backend.javalin; import java.io.InputStream; import java.sql.Connection; import java.util.List; +import java.util.Objects; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -35,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -68,6 +73,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.savedviews.SavedViewsMetaDataProvider; import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; @@ -255,6 +261,9 @@ public class TestUtils .withScriptTypeId(1) .withScriptTester(new QCodeReference(TestScriptAction.class))); + qTableMetaData.withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(PersonTableCustomizer.class)); + qTableMetaData.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(PersonTableCustomizer.class)); + qTableMetaData.getField("photo") .withIsHeavy(true) .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD) @@ -265,6 +274,51 @@ public class TestUtils } + /******************************************************************************* + ** + *******************************************************************************/ + public static class PersonTableCustomizer implements TableCustomizerInterface + { + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + return warnPostInsertOrUpdate(records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + return warnPostInsertOrUpdate(records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List warnPostInsertOrUpdate(List records) + { + for(QRecord record : records) + { + if(Objects.requireNonNullElse(record.getValueString("firstName"), "").toLowerCase().contains("warn")) + { + record.addWarning(new QWarningMessage("Warning in firstName.")); + } + } + + return records; + } + } + + /******************************************************************************* ** diff --git a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java index fe8955eb..2a52e8d7 100644 --- a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java +++ b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java @@ -415,6 +415,7 @@ public class QCommandBuilder { case STRING, TEXT, HTML, PASSWORD -> String.class; case INTEGER -> Integer.class; + case LONG -> Long.class; case DECIMAL -> BigDecimal.class; case DATE -> LocalDate.class; case TIME -> LocalTime.class;