From 455ab69104e50e83b6663264482fa07475a91130 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 22 Dec 2023 18:59:08 -0600 Subject: [PATCH 01/72] Adding QFieldType.LONG --- .../reporting/GenerateReportAction.java | 7 + .../values/QPossibleValueTranslator.java | 4 + .../qqq/backend/core/model/data/QRecord.java | 10 ++ .../model/metadata/fields/QFieldType.java | 7 +- .../memory/MemoryRecordStore.java | 32 ++++- .../implementations/mock/MockQueryAction.java | 1 + .../qqq/backend/core/utils/JsonUtils.java | 1 + .../qqq/backend/core/utils/ValueUtils.java | 108 ++++++++++++++ .../core/utils/aggregates/LongAggregates.java | 135 ++++++++++++++++++ .../rdbms/actions/AbstractRDBMSAction.java | 6 +- .../rdbms/actions/RDBMSAggregateAction.java | 2 +- .../actions/GenerateOpenApiSpecAction.java | 2 +- .../qqq/frontend/picocli/QCommandBuilder.java | 1 + 13 files changed, 309 insertions(+), 7 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java index 5de10c00..8cc0d588 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java @@ -73,6 +73,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.aggregates.AggregatesInterface; import com.kingsrook.qqq.backend.core.utils.aggregates.BigDecimalAggregates; import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates; +import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates; /******************************************************************************* @@ -553,6 +554,12 @@ public class GenerateReportAction AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates()); fieldAggregates.add(record.getValueInteger(field.getName())); } + else if(field.getType().equals(QFieldType.LONG)) + { + @SuppressWarnings("unchecked") + AggregatesInterface fieldAggregates = (AggregatesInterface) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new LongAggregates()); + fieldAggregates.add(record.getValueLong(field.getName())); + } else if(field.getType().equals(QFieldType.DECIMAL)) { @SuppressWarnings("unchecked") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java index 7905e706..782f50d5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java @@ -270,6 +270,10 @@ public class QPossibleValueTranslator { value = ValueUtils.getValueAsInteger(value); } + if(field.getType().equals(QFieldType.LONG) && !(value instanceof Long)) + { + value = ValueUtils.getValueAsLong(value); + } } catch(QValueException e) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index 4457b401..5c4f84ba 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -444,6 +444,16 @@ public class QRecord implements Serializable } + /******************************************************************************* + ** Getter for a single field's value + ** + *******************************************************************************/ + public Long getValueLong(String fieldName) + { + return (ValueUtils.getValueAsLong(values.get(fieldName))); + } + + /******************************************************************************* ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java index 8971a846..b0e8f8e7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java @@ -37,6 +37,7 @@ public enum QFieldType { STRING, INTEGER, + LONG, DECIMAL, BOOLEAN, DATE, @@ -65,6 +66,10 @@ public enum QFieldType { return (INTEGER); } + if(c.equals(Long.class) || c.equals(long.class)) + { + return (LONG); + } if(c.equals(BigDecimal.class)) { return (DECIMAL); @@ -110,7 +115,7 @@ public enum QFieldType *******************************************************************************/ public boolean isNumeric() { - return this == QFieldType.INTEGER || this == QFieldType.DECIMAL; + return this == QFieldType.INTEGER || this == QFieldType.LONG || this == QFieldType.DECIMAL; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java index 4685b7e3..a092a520 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java @@ -366,7 +366,7 @@ public class MemoryRecordStore ///////////////////////////////////////////////// // set the next serial in the record if needed // ///////////////////////////////////////////////// - if(recordToInsert.getValue(primaryKeyField.getName()) == null && primaryKeyField.getType().equals(QFieldType.INTEGER)) + if(recordToInsert.getValue(primaryKeyField.getName()) == null && (primaryKeyField.getType().equals(QFieldType.INTEGER) || primaryKeyField.getType().equals(QFieldType.LONG))) { recordToInsert.setValue(primaryKeyField.getName(), nextSerial++); } @@ -378,6 +378,13 @@ public class MemoryRecordStore { nextSerial = recordToInsert.getValueInteger(primaryKeyField.getName()) + 1; } + else if(primaryKeyField.getType().equals(QFieldType.LONG) && recordToInsert.getValueLong(primaryKeyField.getName()) > nextSerial) + { + ////////////////////////////////////// + // todo - mmm, could overflow here? // + ////////////////////////////////////// + nextSerial = recordToInsert.getValueInteger(primaryKeyField.getName()) + 1; + } tableData.put(recordToInsert.getValue(primaryKeyField.getName()), recordToInsert); if(returnInsertedRecords) @@ -709,7 +716,7 @@ public class MemoryRecordStore { // todo - joins probably? QFieldMetaData field = table.getField(fieldName); - if(field.getType().equals(QFieldType.INTEGER) && (operator.equals(AggregateOperator.AVG))) + if((field.getType().equals(QFieldType.INTEGER) || field.getType().equals(QFieldType.LONG)) && (operator.equals(AggregateOperator.AVG))) { fieldType = QFieldType.DECIMAL; } @@ -745,6 +752,10 @@ public class MemoryRecordStore .filter(r -> r.getValue(fieldName) != null) .mapToInt(r -> r.getValueInteger(fieldName)) .sum(); + case LONG -> records.stream() + .filter(r -> r.getValue(fieldName) != null) + .mapToLong(r -> r.getValueLong(fieldName)) + .sum(); case DECIMAL -> records.stream() .filter(r -> r.getValue(fieldName) != null) .map(r -> r.getValueBigDecimal(fieldName)) @@ -759,6 +770,11 @@ public class MemoryRecordStore .mapToInt(r -> r.getValueInteger(fieldName)) .min() .stream().boxed().findFirst().orElse(null); + case LONG -> records.stream() + .filter(r -> r.getValue(fieldName) != null) + .mapToLong(r -> r.getValueLong(fieldName)) + .min() + .stream().boxed().findFirst().orElse(null); case DECIMAL, STRING, DATE, DATE_TIME -> { Optional serializable = records.stream() @@ -775,7 +791,12 @@ public class MemoryRecordStore { case INTEGER -> records.stream() .filter(r -> r.getValue(fieldName) != null) - .mapToInt(r -> r.getValueInteger(fieldName)) + .mapToLong(r -> r.getValueInteger(fieldName)) + .max() + .stream().boxed().findFirst().orElse(null); + case LONG -> records.stream() + .filter(r -> r.getValue(fieldName) != null) + .mapToLong(r -> r.getValueLong(fieldName)) .max() .stream().boxed().findFirst().orElse(null); case DECIMAL, STRING, DATE, DATE_TIME -> @@ -797,6 +818,11 @@ public class MemoryRecordStore .mapToInt(r -> r.getValueInteger(fieldName)) .average() .stream().boxed().findFirst().orElse(null); + case LONG -> records.stream() + .filter(r -> r.getValue(fieldName) != null) + .mapToLong(r -> r.getValueLong(fieldName)) + .average() + .stream().boxed().findFirst().orElse(null); case DECIMAL -> records.stream() .filter(r -> r.getValue(fieldName) != null) .mapToDouble(r -> r.getValueBigDecimal(fieldName).doubleValue()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java index 9fc87831..970b69ea 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java @@ -103,6 +103,7 @@ public class MockQueryAction implements QueryInterface { case STRING -> UUID.randomUUID().toString(); case INTEGER -> 42; + case LONG -> 42L; case DECIMAL -> new BigDecimal("3.14159"); case DATE -> LocalDate.of(1970, Month.JANUARY, 1); case DATE_TIME -> LocalDateTime.of(1970, Month.JANUARY, 1, 0, 0); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java index a2a36a37..e7d8e9d3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java @@ -362,6 +362,7 @@ public class JsonUtils switch(metaData.getType()) { case INTEGER -> record.setValue(fieldName, jsonObjectToUse.optInt(backendName)); + case LONG -> record.setValue(fieldName, jsonObjectToUse.optLong(backendName)); case DECIMAL -> record.setValue(fieldName, jsonObjectToUse.optBigDecimal(backendName, null)); case BOOLEAN -> record.setValue(fieldName, jsonObjectToUse.optBoolean(backendName)); case DATE_TIME -> diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index dbb1e075..b73fec7d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -114,6 +114,113 @@ public class ValueUtils + /******************************************************************************* + ** Type-safely make an Long from any Object. + ** null and empty-string inputs return null. + ** We try to strip away commas and decimals (as long as they are exactly equal to the int value) + ** We may throw if the input can't be converted to an integer. + *******************************************************************************/ + public static Long getValueAsLong(Object value) throws QValueException + { + try + { + if(value == null) + { + return (null); + } + else if(value instanceof Integer i) + { + return Long.valueOf((i)); + } + else if(value instanceof Long l) + { + return (l); + } + else if(value instanceof BigInteger b) + { + return (b.longValue()); + } + else if(value instanceof Float f) + { + if(f.longValue() != f) + { + throw (new QValueException(f + " does not have an exact integer representation.")); + } + return (f.longValue()); + } + else if(value instanceof Double d) + { + if(d.longValue() != d) + { + throw (new QValueException(d + " does not have an exact integer representation.")); + } + return (d.longValue()); + } + else if(value instanceof BigDecimal bd) + { + return bd.longValueExact(); + } + else if(value instanceof PossibleValueEnum pve) + { + return getValueAsLong(pve.getPossibleValueId()); + } + else if(value instanceof String s) + { + if(!StringUtils.hasContent(s)) + { + return (null); + } + + try + { + return (Long.parseLong(s)); + } + catch(NumberFormatException nfe) + { + if(s.contains(",")) + { + String sWithoutCommas = s.replaceAll(",", ""); + try + { + return (getValueAsLong(sWithoutCommas)); + } + catch(Exception ignore) + { + throw (nfe); + } + } + if(s.matches(".*\\.\\d+$")) + { + String sWithoutDecimal = s.replaceAll("\\.\\d+$", ""); + try + { + return (getValueAsLong(sWithoutDecimal)); + } + catch(Exception ignore) + { + throw (nfe); + } + } + throw (nfe); + } + } + else + { + throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to Long.")); + } + } + catch(QValueException qve) + { + throw (qve); + } + catch(Exception e) + { + throw (new QValueException("Value [" + value + "] could not be converted to a Long.", e)); + } + } + + + /******************************************************************************* ** Type-safely make an Integer from any Object. ** null and empty-string inputs return null. @@ -693,6 +800,7 @@ public class ValueUtils { case STRING, TEXT, HTML, PASSWORD -> getValueAsString(value); case INTEGER -> getValueAsInteger(value); + case LONG -> getValueAsLong(value); case DECIMAL -> getValueAsBigDecimal(value); case BOOLEAN -> getValueAsBoolean(value); case DATE -> getValueAsLocalDate(value); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java new file mode 100644 index 00000000..bcf1862b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/aggregates/LongAggregates.java @@ -0,0 +1,135 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.aggregates; + + +import java.math.BigDecimal; + + +/******************************************************************************* + ** Long version of data aggregator + *******************************************************************************/ +public class LongAggregates implements AggregatesInterface +{ + private int count = 0; + // private Long countDistinct; + private Long sum; + private Long min; + private Long max; + + + + /******************************************************************************* + ** Add a new value to this aggregate set + *******************************************************************************/ + public void add(Long input) + { + if(input == null) + { + return; + } + + count++; + + if(sum == null) + { + sum = input; + } + else + { + sum = sum + input; + } + + if(min == null || input < min) + { + min = input; + } + + if(max == null || input > max) + { + max = input; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int getCount() + { + return (count); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Long getSum() + { + return (sum); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Long getMin() + { + return (min); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Long getMax() + { + return (max); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public BigDecimal getAverage() + { + if(this.count > 0) + { + return (BigDecimal.valueOf(this.sum.doubleValue() / (double) this.count)); + } + else + { + return (null); + } + } + +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index 4aed4e14..ac33f347 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -154,7 +154,7 @@ public abstract class AbstractRDBMSAction implements QActionInterface if("".equals(value)) { QFieldType type = field.getType(); - if(type.equals(QFieldType.INTEGER) || type.equals(QFieldType.DECIMAL) || type.equals(QFieldType.DATE) || type.equals(QFieldType.DATE_TIME) || type.equals(QFieldType.BOOLEAN)) + if(type.equals(QFieldType.INTEGER) || type.equals(QFieldType.LONG) || type.equals(QFieldType.DECIMAL) || type.equals(QFieldType.DATE) || type.equals(QFieldType.DATE_TIME) || type.equals(QFieldType.BOOLEAN)) { value = null; } @@ -875,6 +875,10 @@ public abstract class AbstractRDBMSAction implements QActionInterface { return (QueryManager.getInteger(resultSet, i)); } + case LONG: + { + return (QueryManager.getLong(resultSet, i)); + } case DECIMAL: { return (QueryManager.getBigDecimal(resultSet, i)); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index ce720120..4e0d0fad 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -143,7 +143,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega QFieldType fieldType = aggregate.getFieldType(); if(fieldType == null) { - if(field.getType().equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG))) + if((field.getType().equals(QFieldType.INTEGER) || field.getType().equals(QFieldType.LONG)) && (aggregate.getOperator().equals(AggregateOperator.AVG))) { fieldType = QFieldType.DECIMAL; } diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java index 53617063..2212b13e 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java @@ -1680,7 +1680,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction "string"; - case INTEGER -> "integer"; + case INTEGER, LONG -> "integer"; // todo - we could give 'format' w/ int32 & int64 to further specify case DECIMAL -> "number"; case BOOLEAN -> "boolean"; }; diff --git a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java index fe8955eb..2a52e8d7 100644 --- a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java +++ b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java @@ -415,6 +415,7 @@ public class QCommandBuilder { case STRING, TEXT, HTML, PASSWORD -> String.class; case INTEGER -> Integer.class; + case LONG -> Long.class; case DECIMAL -> BigDecimal.class; case DATE -> LocalDate.class; case TIME -> LocalTime.class; From 7426aa36a51fe138a7426d64a3bbb209b587405d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 22 Dec 2023 18:59:59 -0600 Subject: [PATCH 02/72] Thread name and log cleanups --- .../qqq/backend/core/actions/async/AsyncJobManager.java | 3 ++- .../etl/streamedwithfrontend/BaseStreamedETLStep.java | 6 +++--- .../etl/streamedwithfrontend/StreamedETLExecuteStep.java | 3 +-- .../etl/streamedwithfrontend/StreamedETLPreviewStep.java | 2 +- .../etl/streamedwithfrontend/StreamedETLValidateStep.java | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java index e4871048..ae980ea1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java @@ -159,7 +159,8 @@ public class AsyncJobManager private T runAsyncJob(String jobName, AsyncJob asyncJob, UUIDAndTypeStateKey uuidAndTypeStateKey, AsyncJobStatus asyncJobStatus) { String originalThreadName = Thread.currentThread().getName(); - Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8)); + // Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8)); + Thread.currentThread().setName("Job:" + jobName); try { LOG.debug("Starting job " + uuidAndTypeStateKey.getUuid()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java index 74cab0b6..ec764ce7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java @@ -104,12 +104,12 @@ public class BaseStreamedETLStep *******************************************************************************/ protected void moveReviewStepAfterValidateStep(RunBackendStepOutput runBackendStepOutput) { - LOG.info("Skipping to validation step"); + LOG.debug("Skipping to validation step"); ArrayList stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); - LOG.debug("Step list pre: " + stepList); + LOG.trace("Step list pre: " + stepList); stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)); stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); runBackendStepOutput.getProcessState().setStepList(stepList); - LOG.debug("Step list post: " + stepList); + LOG.trace("Step list post: " + stepList); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java index d842cf03..3790ec07 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java @@ -34,7 +34,6 @@ import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; @@ -128,7 +127,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe asyncRecordPipeLoop.setMinRecordsToConsume(overrideRecordPipeCapacity); } - int recordCount = asyncRecordPipeLoop.run("StreamedETL>Execute>ExtractStep", null, recordPipe, (status) -> + int recordCount = asyncRecordPipeLoop.run("StreamedETLExecute>Extract>" + runBackendStepInput.getProcessName(), null, recordPipe, (status) -> { extractStep.run(runBackendStepInput, runBackendStepOutput); return (runBackendStepOutput); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java index 4921c9ce..e24e617a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -125,7 +125,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe // } List previewRecordList = new ArrayList<>(); - new AsyncRecordPipeLoop().run("StreamedETL>Preview>ExtractStep", PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> + new AsyncRecordPipeLoop().run("StreamedETLPreview>Extract>" + runBackendStepInput.getProcessName(), PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> { runBackendStepInput.setAsyncJobCallback(status); extractStep.run(runBackendStepInput, runBackendStepOutput); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java index 63d858c1..dd8d8022 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java @@ -91,7 +91,7 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back transformStep.preRun(runBackendStepInput, runBackendStepOutput); List previewRecordList = new ArrayList<>(); - int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Preview>ValidateStep", null, recordPipe, (status) -> + int recordCount = new AsyncRecordPipeLoop().run("StreamedETLValidate>Extract>" + runBackendStepInput.getProcessName(), null, recordPipe, (status) -> { extractStep.run(runBackendStepInput, runBackendStepOutput); return (runBackendStepOutput); From 2fc513891f78883b2c78611ed54aaa99dfbfe3aa Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 22 Dec 2023 19:00:31 -0600 Subject: [PATCH 03/72] Add methods allReadCapabilities and allWriteCapabilities (alright) --- .../model/metadata/tables/Capability.java | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java index e5e39c73..aa037868 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java @@ -22,6 +22,9 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables; +import java.util.Set; + + /******************************************************************************* ** Things that can be done to tables, fields. ** @@ -38,5 +41,26 @@ public enum Capability // keep these values in sync with Capability.ts in qqq-frontend-core // /////////////////////////////////////////////////////////////////////// - QUERY_STATS + QUERY_STATS; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Set allReadCapabilities() + { + return (Set.of(TABLE_QUERY, TABLE_GET, TABLE_COUNT, QUERY_STATS)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Set allWriteCapabilities() + { + return (Set.of(TABLE_INSERT, TABLE_UPDATE, TABLE_DELETE)); + } + } From 346443996b7534666caa17f1b715792794f02b62 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 22 Dec 2023 19:02:36 -0600 Subject: [PATCH 04/72] Adding QFieldType.LONG --- .../backend/core/actions/processes/RunBackendStepActionTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java index f012677d..839172d8 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java @@ -103,6 +103,7 @@ public class RunBackendStepActionTest extends BaseTest { case STRING -> "ABC"; case INTEGER -> 42; + case LONG -> 42L; case DECIMAL -> new BigDecimal("47"); case BOOLEAN -> true; case DATE, TIME, DATE_TIME -> null; From 9c7d94f764922274d8478a725c42acf77a448e28 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 22 Dec 2023 19:07:37 -0600 Subject: [PATCH 05/72] Little more user-facing error message --- .../kingsrook/qqq/backend/core/actions/tables/GetAction.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java index 728f5874..6e95bf1a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java @@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; /******************************************************************************* @@ -202,7 +203,7 @@ public class GetAction } else { - throw (new QException("No primaryKey or uniqueKey was passed to Get")); + throw (new QException("Unable to get " + ObjectUtils.tryElse(() -> queryInput.getTable().getLabel(), queryInput.getTableName()) + ". Missing required input.")); } queryInput.setFilter(filter); From 84093dfde55d6232997f9385296d1b5bc880b5ab Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 22 Dec 2023 19:08:43 -0600 Subject: [PATCH 06/72] Fall back to field name if field label isn't set, when giving missing-required-field error --- .../qqq/backend/core/actions/tables/InsertAction.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index a537c883..2d63939f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -286,7 +287,7 @@ public class InsertAction extends AbstractQActionFunction Date: Fri, 22 Dec 2023 19:09:17 -0600 Subject: [PATCH 07/72] Overload withSectionOfChildren that takes Collection instead of varargs --- .../core/model/metadata/layout/QAppMetaData.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java index 0ad7335c..24b46042 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java @@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.layout; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; @@ -357,11 +359,11 @@ public class QAppMetaData implements QAppChildMetaData, MetaDataWithPermissionRu /******************************************************************************* ** *******************************************************************************/ - public QAppMetaData withSectionOfChildren(QAppSection section, QAppChildMetaData... children) + public QAppMetaData withSectionOfChildren(QAppSection section, Collection children) { this.addSection(section); - for(QAppChildMetaData child : children) + for(QAppChildMetaData child : CollectionUtils.nonNullCollection(children)) { withChild(child); if(child instanceof QTableMetaData) @@ -386,6 +388,15 @@ public class QAppMetaData implements QAppChildMetaData, MetaDataWithPermissionRu } + /******************************************************************************* + ** + *******************************************************************************/ + public QAppMetaData withSectionOfChildren(QAppSection section, QAppChildMetaData... children) + { + return (withSectionOfChildren(section, children == null ? null : Arrays.stream(children).toList())); + } + + /******************************************************************************* ** Getter for permissionRules From a37a0b489d79fa6bcb3c349a674427d9a8052617 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 22 Dec 2023 19:09:34 -0600 Subject: [PATCH 08/72] Add a nonNullList around orderBys in toString --- .../backend/core/model/actions/tables/query/QQueryFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java index 6ce122bb..717c8f8d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java @@ -359,7 +359,7 @@ public class QQueryFilter implements Serializable, Cloneable rs.append(")"); rs.append("OrderBy["); - for(QFilterOrderBy orderBy : orderBys) + for(QFilterOrderBy orderBy : CollectionUtils.nonNullList(orderBys)) { rs.append(orderBy).append(","); } From b1e68017ccc80f0f5a93239cd45d3db64883b2ef Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 22 Dec 2023 19:09:58 -0600 Subject: [PATCH 09/72] Fix warn message to have correct name --- .../savedfilters/QuerySavedFilterProcess.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedfilters/QuerySavedFilterProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedfilters/QuerySavedFilterProcess.java index dc50ed17..84556ec4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedfilters/QuerySavedFilterProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedfilters/QuerySavedFilterProcess.java @@ -62,11 +62,9 @@ public class QuerySavedFilterProcess implements BackendStep { return (new QProcessMetaData() .withName("querySavedFilter") - .withStepList(List.of( - new QBackendStepMetaData() - .withCode(new QCodeReference(QuerySavedFilterProcess.class)) - .withName("query") - ))); + .withStepList(List.of(new QBackendStepMetaData() + .withCode(new QCodeReference(QuerySavedFilterProcess.class)) + .withName("query")))); } @@ -110,7 +108,7 @@ public class QuerySavedFilterProcess implements BackendStep } catch(Exception e) { - LOG.warn("Error deleting saved filter", e); + LOG.warn("Error querying for saved filter", e); throw (e); } } From 0dd97d9dc163dd1fc8ac77b5d03ab60fb79bc847 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 22 Dec 2023 19:12:37 -0600 Subject: [PATCH 10/72] Add overload that lets caller customize the jackson object mapper --- .../qqq/backend/core/utils/YamlUtils.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/YamlUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/YamlUtils.java index 1c7a8f53..c52bf027 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/YamlUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/YamlUtils.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.utils; import java.util.Map; +import java.util.function.Consumer; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -56,6 +57,16 @@ public class YamlUtils ** *******************************************************************************/ public static String toYaml(Object object) + { + return toYaml(object, null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String toYaml(Object object, Consumer objectMapperCustomizer) { try { @@ -66,7 +77,10 @@ public class YamlUtils objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); - // todo? objectMapper.setFilterProvider(new OmitDefaultValuesFilterProvider()); + if(objectMapperCustomizer != null) + { + objectMapperCustomizer.accept(objectMapper); + } objectMapper.findAndRegisterModules(); return (objectMapper.writeValueAsString(object)); From 3c2a34291a7ce492f4ee573456f66b1397370a6a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Dec 2023 08:47:03 -0600 Subject: [PATCH 11/72] Refactor, moving methods into SchedulerUtils, for use by other schedulers --- .../core/scheduler/ScheduleManager.java | 130 +------------ .../core/scheduler/SchedulerUtils.java | 176 ++++++++++++++++++ 2 files changed, 182 insertions(+), 124 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java index 375aebb9..cb7e0388 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java @@ -29,20 +29,11 @@ import java.util.Map; import java.util.Objects; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; -import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; -import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.logging.LogPair; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -141,7 +132,7 @@ public class ScheduleManager for(QProcessMetaData process : qInstance.getProcesses().values()) { - if(process.getSchedule() != null && allowedToStart(process.getName())) + if(process.getSchedule() != null && SchedulerUtils.allowedToStart(process.getName())) { QScheduleMetaData scheduleMetaData = process.getSchedule(); if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) @@ -158,7 +149,7 @@ public class ScheduleManager // running at the same time, get the variant records and schedule each separately // ///////////////////////////////////////////////////////////////////////////////////////////////////// QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); - for(QRecord qRecord : CollectionUtils.nonNullList(getBackendVariantFilteredRecords(process))) + for(QRecord qRecord : CollectionUtils.nonNullList(SchedulerUtils.getBackendVariantFilteredRecords(process))) { try { @@ -188,34 +179,6 @@ public class ScheduleManager - /******************************************************************************* - ** - *******************************************************************************/ - private List getBackendVariantFilteredRecords(QProcessMetaData processMetaData) - { - List records = null; - try - { - QScheduleMetaData scheduleMetaData = processMetaData.getSchedule(); - QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); - - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(backendMetaData.getVariantOptionsTableName()); - queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backendMetaData.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backendMetaData.getVariantOptionsTableTypeValue()))); - - QContext.init(qInstance, sessionSupplier.get()); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - records = queryOutput.getRecords(); - } - catch(Exception e) - { - LOG.error("An error fetching variant data for process [" + processMetaData.getLabel() + "]", e); - } - - return (records); - } - - /******************************************************************************* ** @@ -229,7 +192,7 @@ public class ScheduleManager List tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName()); for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions) { - if(allowedToStart(tableAction.tableName())) + if(SchedulerUtils.allowedToStart(tableAction.tableName())) { PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProvider.getName(), sessionSupplier, tableAction); StandardScheduledExecutor executor = new StandardScheduledExecutor(runner); @@ -250,29 +213,12 @@ public class ScheduleManager - /******************************************************************************* - ** - *******************************************************************************/ - private boolean allowedToStart(String name) - { - String propertyName = "qqq.scheduleManager.onlyStartNamesMatching"; - String propertyValue = System.getProperty(propertyName, ""); - if(propertyValue.equals("")) - { - return (true); - } - - return (name.matches(propertyValue)); - } - - - /******************************************************************************* ** *******************************************************************************/ private void startQueueProvider(QQueueProviderMetaData queueProvider) { - if(allowedToStart(queueProvider.getName())) + if(SchedulerUtils.allowedToStart(queueProvider.getName())) { switch(queueProvider.getType()) { @@ -297,7 +243,7 @@ public class ScheduleManager for(QQueueMetaData queue : qInstance.getQueues().values()) { - if(queueProvider.getName().equals(queue.getProviderName()) && allowedToStart(queue.getName())) + if(queueProvider.getName().equals(queue.getProviderName()) && SchedulerUtils.allowedToStart(queue.getName())) { SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); sqsQueuePoller.setQueueProviderMetaData(queueProvider); @@ -332,46 +278,7 @@ public class ScheduleManager { Runnable runProcess = () -> { - String originalThreadName = Thread.currentThread().getName(); - - try - { - if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) - { - QContext.init(qInstance, sessionSupplier.get()); - executeSingleProcess(process, backendVariantData); - } - else if(QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) - { - /////////////////////////////////////////////////////////////////////////////////////////////////// - // if this is "serial", which for example means we want to run each backend variant one after // - // the other in the same thread so loop over these here so that they run in same lambda function // - /////////////////////////////////////////////////////////////////////////////////////////////////// - for(QRecord qRecord : getBackendVariantFilteredRecords(process)) - { - try - { - QContext.init(qInstance, sessionSupplier.get()); - QScheduleMetaData scheduleMetaData = process.getSchedule(); - QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); - executeSingleProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); - } - catch(Exception e) - { - LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); - } - } - } - } - catch(Exception e) - { - LOG.warn("Exception thrown running scheduled process [" + process.getName() + "]", e); - } - finally - { - Thread.currentThread().setName(originalThreadName); - QContext.clear(); - } + SchedulerUtils.runProcess(qInstance, sessionSupplier, process, backendVariantData); }; StandardScheduledExecutor executor = new StandardScheduledExecutor(runProcess); @@ -387,31 +294,6 @@ public class ScheduleManager - /******************************************************************************* - ** - *******************************************************************************/ - private static void executeSingleProcess(QProcessMetaData process, Map backendVariantData) throws QException - { - if(backendVariantData != null) - { - QContext.getQSession().setBackendVariants(backendVariantData); - } - - Thread.currentThread().setName("ScheduledProcess>" + process.getName()); - LOG.debug("Running Scheduled Process [" + process.getName() + "]"); - - RunProcessInput runProcessInput = new RunProcessInput(); - runProcessInput.setProcessName(process.getName()); - runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); - - QContext.pushAction(runProcessInput); - - RunProcessAction runProcessAction = new RunProcessAction(); - runProcessAction.execute(runProcessInput); - } - - - /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java new file mode 100644 index 00000000..76f86394 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java @@ -0,0 +1,176 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.LogPair; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SchedulerUtils +{ + private static final QLogger LOG = QLogger.getLogger(SchedulerUtils.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static boolean allowedToStart(String name) + { + String propertyName = "qqq.scheduleManager.onlyStartNamesMatching"; + String propertyValue = System.getProperty(propertyName, ""); + if(propertyValue.equals("")) + { + return (true); + } + + return (name.matches(propertyValue)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void runProcess(QInstance qInstance, Supplier sessionSupplier, QProcessMetaData process, Map backendVariantData) + { + String originalThreadName = Thread.currentThread().getName(); + + try + { + QContext.init(qInstance, sessionSupplier.get()); + + if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) + { + SchedulerUtils.executeSingleProcess(process, backendVariantData); + } + else if(QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) + { + /////////////////////////////////////////////////////////////////////////////////////////////////// + // if this is "serial", which for example means we want to run each backend variant one after // + // the other in the same thread so loop over these here so that they run in same lambda function // + /////////////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord qRecord : getBackendVariantFilteredRecords(process)) + { + try + { + QScheduleMetaData scheduleMetaData = process.getSchedule(); + QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); + executeSingleProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); + } + catch(Exception e) + { + LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); + } + } + } + } + catch(Exception e) + { + LOG.warn("Exception thrown running scheduled process [" + process.getName() + "]", e); + } + finally + { + Thread.currentThread().setName(originalThreadName); + QContext.clear(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void executeSingleProcess(QProcessMetaData process, Map backendVariantData) throws QException + { + if(backendVariantData != null) + { + QContext.getQSession().setBackendVariants(backendVariantData); + } + + Thread.currentThread().setName("ScheduledProcess>" + process.getName()); + LOG.debug("Running Scheduled Process [" + process.getName() + "]"); + + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(process.getName()); + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + + QContext.pushAction(runProcessInput); + + RunProcessAction runProcessAction = new RunProcessAction(); + runProcessAction.execute(runProcessInput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static List getBackendVariantFilteredRecords(QProcessMetaData processMetaData) + { + List records = null; + try + { + QScheduleMetaData scheduleMetaData = processMetaData.getSchedule(); + QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(scheduleMetaData.getVariantBackend()); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(backendMetaData.getVariantOptionsTableName()); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backendMetaData.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backendMetaData.getVariantOptionsTableTypeValue()))); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + records = queryOutput.getRecords(); + } + catch(Exception e) + { + LOG.error("An error fetching variant data for process [" + processMetaData.getLabel() + "]", e); + } + + return (records); + } + +} From 1ee4f67286e97b394cf1ad70ea234eb4765516aa Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Dec 2023 08:46:32 -0600 Subject: [PATCH 12/72] Initial quartz scheduler and processes --- qqq-backend-core/pom.xml | 11 + .../scheduler/quartz/QuartzRunProcessJob.java | 77 ++++ .../scheduler/quartz/QuartzScheduler.java | 365 ++++++++++++++++++ .../processes/PauseAllQuartzJobsProcess.java | 86 +++++ .../processes/PauseQuartzJobsProcess.java | 89 +++++ .../processes/ResumeAllQuartzJobsProcess.java | 86 +++++ .../processes/ResumeQuartzJobsProcess.java | 89 +++++ .../src/main/resources/quartz.properties | 64 +++ 8 files changed, 867 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java create mode 100644 qqq-backend-core/src/main/resources/quartz.properties diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index 58191fa6..f8da0a96 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -163,6 +163,17 @@ 1.12.321 + + org.quartz-scheduler + quartz + 2.3.2 + + + org.slf4j + slf4j-api + 2.0.9 + + org.apache.maven.plugins diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java new file mode 100644 index 00000000..a5bb27f3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import java.io.Serializable; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +@DisallowConcurrentExecution +public class QuartzRunProcessJob implements Job +{ + private static final QLogger LOG = QLogger.getLogger(QuartzRunProcessJob.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException + { + try + { + JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); + String processName = jobDataMap.getString("processName"); + + /////////////////////////////////// + // todo - variants from job data // + /////////////////////////////////// + Map backendVariantData = null; + + LOG.debug("Running quartz process", logPair("processName", processName)); + + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + SchedulerUtils.runProcess(qInstance, QuartzScheduler.getInstance().getSessionSupplier(), qInstance.getProcess(processName), backendVariantData); + + } + finally + { + QContext.clear(); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java new file mode 100644 index 00000000..5a8385f6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java @@ -0,0 +1,365 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import java.io.Serializable; +import java.util.Date; +import java.util.Map; +import java.util.Objects; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; +import org.quartz.JobBuilder; +import org.quartz.JobDetail; +import org.quartz.JobKey; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.SimpleScheduleBuilder; +import org.quartz.Trigger; +import org.quartz.TriggerBuilder; +import org.quartz.TriggerKey; +import org.quartz.impl.StdSchedulerFactory; +import org.quartz.impl.matchers.GroupMatcher; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuartzScheduler +{ + private static final QLogger LOG = QLogger.getLogger(QuartzScheduler.class); + + private static QuartzScheduler quartzScheduler = null; + + private final QInstance qInstance; + private Supplier sessionSupplier; + + private Scheduler scheduler; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + private QuartzScheduler(QInstance qInstance, Supplier sessionSupplier) + { + this.qInstance = qInstance; + this.sessionSupplier = sessionSupplier; + } + + + + /******************************************************************************* + ** Singleton initiator... + *******************************************************************************/ + public static QuartzScheduler initInstance(QInstance qInstance, Supplier sessionSupplier) + { + if(quartzScheduler == null) + { + quartzScheduler = new QuartzScheduler(qInstance, sessionSupplier); + } + return (quartzScheduler); + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static QuartzScheduler getInstance() + { + if(quartzScheduler == null) + { + throw (new IllegalStateException("QuartzScheduler singleton has not been init'ed.")); + } + return (quartzScheduler); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void start() + { + try + { + // Properties properties = new Properties(); + // properties.put(""); + + ////////////////////////////////////////////////// + // Grab the Scheduler instance from the Factory // + ////////////////////////////////////////////////// + StdSchedulerFactory schedulerFactory = new StdSchedulerFactory(); + // schedulerFactory.initialize(properties); + this.scheduler = schedulerFactory.getScheduler(); + + //////////////////////////////////////// + // todo - do we get our own property? // + //////////////////////////////////////// + if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) + { + LOG.info("Not starting QuartzScheduler per settings."); + return; + } + + ///////////////////////////////////////////// + // make sure all of our jobs are scheduled // + ///////////////////////////////////////////// + scheduleAllJobs(); + + ////////////////////// + // and start it off // + ////////////////////// + scheduler.start(); + } + catch(Exception e) + { + LOG.error("Error starting quartz scheduler", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void scheduleAllJobs() + { + for(QProcessMetaData process : qInstance.getProcesses().values()) + { + if(process.getSchedule() != null && SchedulerUtils.allowedToStart(process.getName())) + { + if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) + { + scheduleProcess(process, null); + } + else + { + LOG.error("Not yet know how to schedule parallel variant jobs"); + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void scheduleProcess(QProcessMetaData process, Map backendVariantData) + { + try + { + QScheduleMetaData scheduleMetaData = process.getSchedule(); + long intervalMillis = Objects.requireNonNullElse(scheduleMetaData.getRepeatMillis(), scheduleMetaData.getRepeatSeconds() * 1000); + + Date startAt = new Date(); + if(scheduleMetaData.getInitialDelayMillis() != null) + { + startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelayMillis()); + } + if(scheduleMetaData.getInitialDelaySeconds() != null) + { + startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelaySeconds() * 1000); + } + + ///////////////////////// + // Define job instance // + ///////////////////////// + JobKey jobKey = new JobKey(process.getName(), "processes"); + JobDetail jobDetail = JobBuilder.newJob(QuartzRunProcessJob.class) + .withIdentity(jobKey) + .storeDurably() + .requestRecovery() + .build(); + + jobDetail.getJobDataMap().put("processName", process.getName()); + + /////////////////////////////////////// + // Define a Trigger for the schedule // + /////////////////////////////////////// + Trigger trigger = TriggerBuilder.newTrigger() + .withIdentity(new TriggerKey(process.getName(), "processes")) + .forJob(jobKey) + .withSchedule(SimpleScheduleBuilder.simpleSchedule() + .withIntervalInMilliseconds(intervalMillis) + .repeatForever()) + .startAt(startAt) + .build(); + + /////////////////////////////////////// + // Schedule the job with the trigger // + /////////////////////////////////////// + boolean isJobAlreadyScheduled = isJobAlreadyScheduled(jobKey); + if(isJobAlreadyScheduled) + { + this.scheduler.addJob(jobDetail, true); + this.scheduler.rescheduleJob(trigger.getKey(), trigger); + LOG.info("Re-scheduled process: " + process.getName()); + } + else + { + this.scheduler.scheduleJob(jobDetail, trigger); + LOG.info("Scheduled new process: " + process.getName()); + } + + } + catch(Exception e) + { + LOG.warn("Error scheduling process", e, logPair("processName", process.getName())); + } + } + + + + /******************************************************************************* + ** todo - probably rewrite this to not re-query quartz each time + *******************************************************************************/ + private boolean isJobAlreadyScheduled(JobKey jobKey) throws SchedulerException + { + for(String group : scheduler.getJobGroupNames()) + { + for(JobKey testJobKey : scheduler.getJobKeys(GroupMatcher.groupEquals(group))) + { + if(testJobKey.equals(jobKey)) + { + return (true); + } + } + } + + return (false); + } + + + + /* + private void todo() throws SchedulerException + { + // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/ListJobs.html + // Listing all Jobs in the scheduler + for(String group : scheduler.getJobGroupNames()) + { + for(JobKey jobKey : scheduler.getJobKeys(GroupMatcher.groupEquals(group))) + { + System.out.println("Found job identified by: " + jobKey); + } + } + + // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UpdateJob.html + // Update an existing job + // Add the new job to the scheduler, instructing it to "replace" + // the existing job with the given name and group (if any) + JobDetail jobDetail = JobBuilder.newJob(QuartzRunProcessJob.class) + .withIdentity("job1", "group1") + .build(); + // store, and set overwrite flag to 'true' + scheduler.addJob(jobDetail, true); + + // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UpdateTrigger.html + // Define a new Trigger + Trigger trigger = TriggerBuilder.newTrigger() + .withIdentity("newTrigger", "group1") + .startNow() + .build(); + + // tell the scheduler to remove the old trigger with the given key, and put the new one in its place + scheduler.rescheduleJob(new TriggerKey("oldTrigger", "group1"), trigger); + + // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UnscheduleJob.html + // Deleting a Job and Unscheduling All of Its Triggers + scheduler.deleteJob(new JobKey("job1", "group1")); + + } + */ + + + + /******************************************************************************* + ** Getter for qInstance + ** + *******************************************************************************/ + public QInstance getQInstance() + { + return qInstance; + } + + + + /******************************************************************************* + ** Getter for sessionSupplier + ** + *******************************************************************************/ + public Supplier getSessionSupplier() + { + return sessionSupplier; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void pauseAll() throws SchedulerException + { + this.scheduler.pauseAll(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void resumeAll() throws SchedulerException + { + this.scheduler.resumeAll(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void pauseJob(String jobName, String groupName) throws SchedulerException + { + this.scheduler.pauseJob(new JobKey(jobName, groupName)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void resumeJob(String jobName, String groupName) throws SchedulerException + { + this.scheduler.resumeJob(new JobKey(jobName, groupName)); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java new file mode 100644 index 00000000..7d820f08 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class PauseAllQuartzJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Pause All Quartz Jobs") + .withStepList(List.of( + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All quartz jobs have been paused"))))) + .withIcon(new QIcon("pause_circle_outline")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QuartzScheduler.getInstance().pauseAll(); + } + catch(Exception e) + { + throw (new QException("Error pausing all jobs", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java new file mode 100644 index 00000000..0aaff30a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java @@ -0,0 +1,89 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.NoopTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class PauseQuartzJobsProcess extends AbstractLoadStep implements MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + String tableName = "QUARTZ_TRIGGERS"; + + return StreamedETLWithFrontendProcess.processMetaDataBuilder() + .withName(getClass().getSimpleName()) + .withLabel("Pause Quartz Jobs") + .withTableName(tableName) + .withSourceTable(tableName) + .withDestinationTable(tableName) + .withExtractStepClass(ExtractViaQueryStep.class) + .withTransformStepClass(NoopTransformStep.class) + .withLoadStepClass(getClass()) + .withIcon(new QIcon("pause_circle_outline")) + .getProcessMetaData(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QuartzScheduler instance = QuartzScheduler.getInstance(); + for(QRecord record : runBackendStepInput.getRecords()) + { + instance.pauseJob(record.getValueString("JOB_NAME"), record.getValueString("GROUP_NAME")); + } + } + catch(Exception e) + { + throw (new QException("Error pausing jobs", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java new file mode 100644 index 00000000..9cd769c7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ResumeAllQuartzJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Resume All Quartz Jobs") + .withStepList(List.of( + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All quartz jobs have been resumed"))))) + .withIcon(new QIcon("play_circle_outline")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QuartzScheduler.getInstance().resumeAll(); + } + catch(Exception e) + { + throw (new QException("Error resuming all jobs", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java new file mode 100644 index 00000000..c81ccf32 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java @@ -0,0 +1,89 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.NoopTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ResumeQuartzJobsProcess extends AbstractLoadStep implements MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + String tableName = "QUARTZ_TRIGGERS"; + + return StreamedETLWithFrontendProcess.processMetaDataBuilder() + .withName(getClass().getSimpleName()) + .withLabel("Resume Quartz Jobs") + .withTableName(tableName) + .withSourceTable(tableName) + .withDestinationTable(tableName) + .withExtractStepClass(ExtractViaQueryStep.class) + .withTransformStepClass(NoopTransformStep.class) + .withLoadStepClass(getClass()) + .withIcon(new QIcon("play_circle_outline")) + .getProcessMetaData(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QuartzScheduler instance = QuartzScheduler.getInstance(); + for(QRecord record : runBackendStepInput.getRecords()) + { + instance.resumeJob(record.getValueString("JOB_NAME"), record.getValueString("GROUP_NAME")); + } + } + catch(Exception e) + { + throw (new QException("Error resuming jobs", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/resources/quartz.properties b/qqq-backend-core/src/main/resources/quartz.properties new file mode 100644 index 00000000..3dd81314 --- /dev/null +++ b/qqq-backend-core/src/main/resources/quartz.properties @@ -0,0 +1,64 @@ +# +# QQQ - Low-code Application Framework for Engineers. +# Copyright (C) 2021-2023. Kingsrook, LLC +# 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States +# contact@kingsrook.com +# https://github.com/Kingsrook/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# +#org.quartz.scheduler.instanceName = MyScheduler +#org.quartz.threadPool.threadCount = 3 +#org.quartz.jobStore.class = org.quartz.simpl.RAMJobStore +# + + + +#============================================================================ +# Configure Main Scheduler Properties +#============================================================================ +org.quartz.scheduler.instanceName = MyClusteredScheduler +org.quartz.scheduler.instanceId = AUTO + +#============================================================================ +# Configure ThreadPool +#============================================================================ +org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool +org.quartz.threadPool.threadCount = 5 +org.quartz.threadPool.threadPriority = 5 + +#============================================================================ +# Configure JobStore +#============================================================================ +org.quartz.jobStore.misfireThreshold = 60000 + +org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX +org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate +org.quartz.jobStore.useProperties = false +org.quartz.jobStore.dataSource = myDS +org.quartz.jobStore.tablePrefix = QUARTZ_ + +org.quartz.jobStore.isClustered = true +org.quartz.jobStore.clusterCheckinInterval = 20000 + +#============================================================================ +# Configure Datasources +#============================================================================ +org.quartz.dataSource.myDS.driver = com.mysql.cj.jdbc.Driver +org.quartz.dataSource.myDS.URL = jdbc:mysql://localhost:3306/nutrifresh_one +org.quartz.dataSource.myDS.user = root +org.quartz.dataSource.myDS.password = BXca6Bubxf!ECt7sua6L +org.quartz.dataSource.myDS.maxConnections = 5 +org.quartz.dataSource.myDS.validationQuery=select 1 \ No newline at end of file From 51945aa844c06513667c67d6a8e2b603f2d6fec5 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Dec 2023 08:47:38 -0600 Subject: [PATCH 13/72] Initial checkin --- .../metadata/RDBMSTableMetaDataBuilder.java | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java new file mode 100644 index 00000000..524712b1 --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java @@ -0,0 +1,162 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.model.metadata; + + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; + + +/******************************************************************************* + ** note - this class is pretty mysql-specific + *******************************************************************************/ +public class RDBMSTableMetaDataBuilder +{ + private static final QLogger LOG = QLogger.getLogger(RDBMSTableMetaDataBuilder.class); + + private static Map typeMap = new HashMap<>(); + + static + { + //////////////////////////////////////////////// + // these are types as returned by mysql // + // let null in here mean unsupported QQQ type // + //////////////////////////////////////////////// + typeMap.put("TEXT", QFieldType.TEXT); + typeMap.put("BINARY", QFieldType.BLOB); + typeMap.put("SET", null); + typeMap.put("VARBINARY", QFieldType.BLOB); + typeMap.put("MEDIUMBLOB", QFieldType.BLOB); + typeMap.put("NUMERIC", QFieldType.INTEGER); + typeMap.put("BIGINT UNSIGNED", QFieldType.LONG); + typeMap.put("MEDIUMINT UNSIGNED", QFieldType.INTEGER); + typeMap.put("SMALLINT UNSIGNED", QFieldType.INTEGER); + typeMap.put("TINYINT UNSIGNED", QFieldType.INTEGER); + typeMap.put("BIT", null); + typeMap.put("FLOAT", null); + typeMap.put("REAL", null); + typeMap.put("VARCHAR", QFieldType.STRING); + typeMap.put("BOOL", QFieldType.BOOLEAN); + typeMap.put("YEAR", null); + typeMap.put("TIME", QFieldType.TIME); + typeMap.put("TIMESTAMP", QFieldType.DATE_TIME); + } + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData buildTableMetaData(RDBMSBackendMetaData backendMetaData, String tableName) throws QException + { + try(Connection connection = new ConnectionManager().getConnection(backendMetaData)) + { + List fieldMetaDataList = new ArrayList<>(); + String primaryKey = null; + + DatabaseMetaData databaseMetaData = connection.getMetaData(); + Map dataTypeMap = new HashMap<>(); + ResultSet typeInfoResultSet = databaseMetaData.getTypeInfo(); + while(typeInfoResultSet.next()) + { + String name = typeInfoResultSet.getString("TYPE_NAME"); + Integer id = typeInfoResultSet.getInt("DATA_TYPE"); + dataTypeMap.put(id, name); + } + + String databaseName = backendMetaData.getDatabaseName(); // these work for mysql - unclear about other vendors. + String schemaName = null; + try(ResultSet tableResultSet = databaseMetaData.getTables(databaseName, schemaName, tableName, null)) + { + if(!tableResultSet.next()) + { + throw (new QException("Table: " + tableName + " was not found in backend: " + backendMetaData.getName())); + } + } + + try(ResultSet columnsResultSet = databaseMetaData.getColumns(databaseName, schemaName, tableName, null)) + { + while(columnsResultSet.next()) + { + String columnName = columnsResultSet.getString("COLUMN_NAME"); + String columnSize = columnsResultSet.getString("COLUMN_SIZE"); + Integer dataTypeId = columnsResultSet.getInt("DATA_TYPE"); + String isNullable = columnsResultSet.getString("IS_NULLABLE"); + String isAutoIncrement = columnsResultSet.getString("IS_AUTOINCREMENT"); + + String dataTypeName = dataTypeMap.get(dataTypeId); + QFieldType type = typeMap.get(dataTypeName); + if(type == null) + { + LOG.info("Table " + tableName + " column " + columnName + " has an unampped type: " + dataTypeId + ". Field will not be added to QTableMetaData"); + continue; + } + + QFieldMetaData fieldMetaData = new QFieldMetaData(columnName, type) + // todo - what string? .withIsRequired(!isNullable) + .withLabel(columnName); + + fieldMetaDataList.add(fieldMetaData); + + if("YES".equals(isAutoIncrement)) + { + primaryKey = columnName; + } + } + } + + if(fieldMetaDataList.isEmpty()) + { + throw (new QException("Could not find any usable fields in table: " + tableName)); + } + + if(primaryKey == null) + { + throw (new QException("Could not find primary key in table: " + tableName)); + } + + QTableMetaData tableMetaData = new QTableMetaData() + .withBackendName(backendMetaData.getName()) + .withName(tableName) + .withLabel(tableName) + .withBackendDetails(new RDBMSTableBackendDetails().withTableName(tableName)) + .withFields(fieldMetaDataList) + .withPrimaryKeyField(primaryKey); + + return (tableMetaData); + } + catch(Exception e) + { + throw (new QException("Error automatically building table meta data for table: " + tableName, e)); + } + } + +} From 12eb1804adcc3d88f89ebfc6fc3318ea5ac0c2f7 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:41:52 -0500 Subject: [PATCH 14/72] CE-936 - Add getName to TopLevelMetaDataInterface --- .../model/metadata/TopLevelMetaDataInterface.java | 5 +++++ .../model/metadata/branding/QBrandingMetaData.java | 11 +++++++++++ .../model/metadata/ApiInstanceMetaDataContainer.java | 11 +++++++++++ 3 files changed, 27 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/TopLevelMetaDataInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/TopLevelMetaDataInterface.java index 7be8db54..e3dc117f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/TopLevelMetaDataInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/TopLevelMetaDataInterface.java @@ -29,6 +29,11 @@ package com.kingsrook.qqq.backend.core.model.metadata; public interface TopLevelMetaDataInterface { + /******************************************************************************* + ** + *******************************************************************************/ + String getName(); + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java index 6e2d7541..6eb01b58 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java @@ -55,6 +55,17 @@ public class QBrandingMetaData implements TopLevelMetaDataInterface + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getName() + { + return "Branding"; + } + + + /******************************************************************************* ** Getter for companyName ** diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaDataContainer.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaDataContainer.java index 71fcac0a..2ea76133 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaDataContainer.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaDataContainer.java @@ -40,6 +40,17 @@ public class ApiInstanceMetaDataContainer extends QSupplementalInstanceMetaData + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getName() + { + return ApiSupplementType.NAME; + } + + + /******************************************************************************* ** Constructor ** From 87ef20aff41ad4d22d9c19224f9f2cabb7d4dc24 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:42:26 -0500 Subject: [PATCH 15/72] CE-936 - Wrap sets w/ HashSets, for mutability. --- .../qqq/backend/core/model/metadata/tables/Capability.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java index aa037868..f272620f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Capability.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables; +import java.util.HashSet; import java.util.Set; @@ -50,7 +51,7 @@ public enum Capability *******************************************************************************/ public static Set allReadCapabilities() { - return (Set.of(TABLE_QUERY, TABLE_GET, TABLE_COUNT, QUERY_STATS)); + return (new HashSet<>(Set.of(TABLE_QUERY, TABLE_GET, TABLE_COUNT, QUERY_STATS))); } @@ -60,7 +61,7 @@ public enum Capability *******************************************************************************/ public static Set allWriteCapabilities() { - return (Set.of(TABLE_INSERT, TABLE_UPDATE, TABLE_DELETE)); + return (new HashSet<>(Set.of(TABLE_INSERT, TABLE_UPDATE, TABLE_DELETE))); } } From 621997efd968af3378b973fd9b7b34cfe0d2559a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:42:45 -0500 Subject: [PATCH 16/72] CE-936 - Add alternate constructor that takes just QInstance and QSession --- .../kingsrook/qqq/backend/core/context/CapturedContext.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java index 1d1a52bb..b1a80fd5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java @@ -34,5 +34,8 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; *******************************************************************************/ public record CapturedContext(QInstance qInstance, QSession qSession, QBackendTransaction qBackendTransaction, Stack actionStack) { - + public CapturedContext(QInstance qInstance, QSession qSession) + { + this(qInstance, qSession, null, null); + } } From 2e1bf399f9d9b5e7ccc01cd655163db4392bc8bb Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:45:36 -0500 Subject: [PATCH 17/72] CE-936 - Add methods getJdbcDriverClassName, getJdbcUrl --- .../module/rdbms/jdbc/ConnectionManager.java | 41 +++++++++++++++++++ .../model/metadata/RDBMSBackendMetaData.java | 32 +++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java index 6714979b..ec62895b 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java @@ -41,10 +41,41 @@ public class ConnectionManager public Connection getConnection(RDBMSBackendMetaData backend) throws SQLException { String jdbcURL; + String jdbcURL = getJdbcUrl(backend); + return DriverManager.getConnection(jdbcURL, backend.getUsername(), backend.getPassword()); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getJdbcDriverClassName(RDBMSBackendMetaData backend) + { + if(StringUtils.hasContent(backend.getJdbcDriverClassName())) + { + return backend.getJdbcDriverClassName(); + } + + return switch(backend.getVendor()) + { + case "mysql", "aurora" -> "com.mysql.cj.jdbc.Driver"; + case "h2" -> "org.h2.Driver"; + default -> throw (new IllegalStateException("We do not know what jdbc driver to use for vendor name [" + backend.getVendor() + "]. Try setting jdbcDriverClassName in your backend meta data.")); + }; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getJdbcUrl(RDBMSBackendMetaData backend) + { if(StringUtils.hasContent(backend.getJdbcUrl())) { jdbcURL = backend.getJdbcUrl(); + return backend.getJdbcUrl(); } else { @@ -60,6 +91,16 @@ public class ConnectionManager } return DriverManager.getConnection(jdbcURL, backend.getUsername(), backend.getPassword()); + + return switch(backend.getVendor()) + { + // TODO aws-mysql-jdbc driver not working when running on AWS + // jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=CONVERT_TO_NULL"; + case "aurora" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false"; + case "mysql" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull"; + case "h2" -> "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1"; + default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor()); + }; } } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java index 6ecc6e8c..a86a6f45 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java @@ -40,6 +40,7 @@ public class RDBMSBackendMetaData extends QBackendMetaData private String password; private String jdbcUrl; + private String jdbcDriverClassName; @@ -314,4 +315,35 @@ public class RDBMSBackendMetaData extends QBackendMetaData return (this); } + + /******************************************************************************* + ** Getter for jdbcDriverClassName + *******************************************************************************/ + public String getJdbcDriverClassName() + { + return (this.jdbcDriverClassName); + } + + + + /******************************************************************************* + ** Setter for jdbcDriverClassName + *******************************************************************************/ + public void setJdbcDriverClassName(String jdbcDriverClassName) + { + this.jdbcDriverClassName = jdbcDriverClassName; + } + + + + /******************************************************************************* + ** Fluent setter for jdbcDriverClassName + *******************************************************************************/ + public RDBMSBackendMetaData withJdbcDriverClassName(String jdbcDriverClassName) + { + this.jdbcDriverClassName = jdbcDriverClassName; + return (this); + } + + } From f448cff5dd3fc0973fdd389cc7be034ab843a71a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:47:21 -0500 Subject: [PATCH 18/72] CE-936 - add setting to useCamelCaseNames --- .../metadata/RDBMSTableMetaDataBuilder.java | 45 +++++++++++++++++-- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java index 524712b1..3d1e774f 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; @@ -44,6 +45,8 @@ public class RDBMSTableMetaDataBuilder { private static final QLogger LOG = QLogger.getLogger(RDBMSTableMetaDataBuilder.class); + private boolean useCamelCaseNames = true; + private static Map typeMap = new HashMap<>(); static @@ -120,9 +123,11 @@ public class RDBMSTableMetaDataBuilder continue; } - QFieldMetaData fieldMetaData = new QFieldMetaData(columnName, type) + String qqqFieldName = QInstanceEnricher.inferNameFromBackendName(columnName); + + QFieldMetaData fieldMetaData = new QFieldMetaData(qqqFieldName, type) // todo - what string? .withIsRequired(!isNullable) - .withLabel(columnName); + .withBackendName(columnName); fieldMetaDataList.add(fieldMetaData); @@ -143,10 +148,11 @@ public class RDBMSTableMetaDataBuilder throw (new QException("Could not find primary key in table: " + tableName)); } + String qqqTableName = QInstanceEnricher.inferNameFromBackendName(tableName); + QTableMetaData tableMetaData = new QTableMetaData() .withBackendName(backendMetaData.getName()) - .withName(tableName) - .withLabel(tableName) + .withName(qqqTableName) .withBackendDetails(new RDBMSTableBackendDetails().withTableName(tableName)) .withFields(fieldMetaDataList) .withPrimaryKeyField(primaryKey); @@ -159,4 +165,35 @@ public class RDBMSTableMetaDataBuilder } } + + /******************************************************************************* + ** Getter for useCamelCaseNames + *******************************************************************************/ + public boolean getUseCamelCaseNames() + { + return (this.useCamelCaseNames); + } + + + + /******************************************************************************* + ** Setter for useCamelCaseNames + *******************************************************************************/ + public void setUseCamelCaseNames(boolean useCamelCaseNames) + { + this.useCamelCaseNames = useCamelCaseNames; + } + + + + /******************************************************************************* + ** Fluent setter for useCamelCaseNames + *******************************************************************************/ + public RDBMSTableMetaDataBuilder withUseCamelCaseNames(boolean useCamelCaseNames) + { + this.useCamelCaseNames = useCamelCaseNames; + return (this); + } + + } From 58c15e6eaaaa020bacfe5b5a111558308525df13 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:48:24 -0500 Subject: [PATCH 19/72] CE-936 - Update to not re-do post-actions if using the defaultGetInterface (which does a query) --- .../qqq/backend/core/actions/tables/GetAction.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java index 6e95bf1a..8c19de56 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java @@ -108,9 +108,11 @@ public class GetAction } GetOutput getOutput; + boolean usingDefaultGetInterface = false; if(getInterface == null) { getInterface = new DefaultGetInterface(); + usingDefaultGetInterface = true; } getInterface.validateInput(getInput); @@ -124,10 +126,11 @@ public class GetAction new GetActionCacheHelper().handleCaching(getInput, getOutput); } - //////////////////////////////////////////////////////// - // if the record is found, perform post-actions on it // - //////////////////////////////////////////////////////// - if(getOutput.getRecord() != null) + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the record is found, perform post-actions on it // + // unless the defaultGetInteface was used - as it just does a query, and the query will do the post-actions. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(getOutput.getRecord() != null && !usingDefaultGetInterface) { getOutput.setRecord(postRecordActions(getOutput.getRecord())); } From 5564e94ad7cb083719b4f3b8a731533a7112178f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:48:48 -0500 Subject: [PATCH 20/72] CE-936 - Set c3p0 (com.mchange) and quartz to INFO level --- qqq-backend-core/src/main/resources/log4j2.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qqq-backend-core/src/main/resources/log4j2.xml b/qqq-backend-core/src/main/resources/log4j2.xml index 349d47d1..ffb3baa6 100644 --- a/qqq-backend-core/src/main/resources/log4j2.xml +++ b/qqq-backend-core/src/main/resources/log4j2.xml @@ -26,6 +26,11 @@ + + + + + From 3e604f4b6feafaa1f98de1f274bf4c49c415fd99 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:49:04 -0500 Subject: [PATCH 21/72] CE-936 - Update for camelCase quartz table names --- .../core/scheduler/quartz/processes/PauseQuartzJobsProcess.java | 2 +- .../scheduler/quartz/processes/ResumeQuartzJobsProcess.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java index 0aaff30a..fbaa4679 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java @@ -49,7 +49,7 @@ public class PauseQuartzJobsProcess extends AbstractLoadStep implements MetaData @Override public QProcessMetaData produce(QInstance qInstance) throws QException { - String tableName = "QUARTZ_TRIGGERS"; + String tableName = "quartzTriggers"; return StreamedETLWithFrontendProcess.processMetaDataBuilder() .withName(getClass().getSimpleName()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java index c81ccf32..a4ceff24 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java @@ -49,7 +49,7 @@ public class ResumeQuartzJobsProcess extends AbstractLoadStep implements MetaDat @Override public QProcessMetaData produce(QInstance qInstance) throws QException { - String tableName = "QUARTZ_TRIGGERS"; + String tableName = "quartzTriggers"; return StreamedETLWithFrontendProcess.processMetaDataBuilder() .withName(getClass().getSimpleName()) From 891c567a8daec0a177e78a632f1e0a4fa708a5cf Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:49:51 -0500 Subject: [PATCH 22/72] CE-936 - update getAllAvailablePermissions, for table permissions, to only include them based on table capabilities --- .../permissions/PermissionsHelper.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java index 39090e2f..cdc2f439 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; @@ -43,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -333,9 +335,25 @@ public class PermissionsHelper QPermissionRules rules = getEffectivePermissionRules(tableMetaData, instance); String baseName = getEffectivePermissionBaseName(rules, tableMetaData.getName()); - for(TablePermissionSubType permissionSubType : TablePermissionSubType.values()) + QBackendMetaData backend = instance.getBackend(tableMetaData.getBackendName()); + if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_INSERT)) { - addEffectiveAvailablePermission(rules, permissionSubType, rs, baseName, tableMetaData, "Table"); + addEffectiveAvailablePermission(rules, TablePermissionSubType.INSERT, rs, baseName, tableMetaData, "Table"); + } + + if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_UPDATE)) + { + addEffectiveAvailablePermission(rules, TablePermissionSubType.EDIT, rs, baseName, tableMetaData, "Table"); + } + + if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_DELETE)) + { + addEffectiveAvailablePermission(rules, TablePermissionSubType.DELETE, rs, baseName, tableMetaData, "Table"); + } + + if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_QUERY) || tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_GET)) + { + addEffectiveAvailablePermission(rules, TablePermissionSubType.READ, rs, baseName, tableMetaData, "Table"); } } From d551ad71a69db75404526d60734ca2d9d003d4aa Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:50:09 -0500 Subject: [PATCH 23/72] CE-936 - Remove slf4j-api (originally added when quartz was added) --- qqq-backend-core/pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index f8da0a96..d7c1e4cf 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -168,11 +168,6 @@ quartz 2.3.2 - - org.slf4j - slf4j-api - 2.0.9 - From 03f1fc1436de167b9e1caedcf1223efda0df567b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:51:10 -0500 Subject: [PATCH 24/72] CE-936 - Add method withTemporaryContext --- .../qqq/backend/core/context/QContext.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java index 69cb0dbd..c0ca377c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.lambdas.VoidVoidMethod; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -54,6 +55,7 @@ public class QContext private static ThreadLocal> objectsThreadLocal = new ThreadLocal<>(); + /******************************************************************************* ** private constructor - class is not meant to be instantiated. *******************************************************************************/ @@ -105,6 +107,25 @@ public class QContext + /******************************************************************************* + ** + *******************************************************************************/ + public static void withTemporaryContext(CapturedContext context, VoidVoidMethod method) + { + CapturedContext originalContext = QContext.capture(); + try + { + QContext.init(context); + method.run(); + } + finally + { + QContext.init(originalContext); + } + } + + + /******************************************************************************* ** Init a new thread with the context captured from a different thread. e.g., ** when starting some async task. @@ -267,6 +288,7 @@ public class QContext } + /******************************************************************************* ** get one named object from the Context for the current thread. may return null. *******************************************************************************/ @@ -280,6 +302,7 @@ public class QContext } + /******************************************************************************* ** get one named object from the Context for the current thread, cast to the ** specified type if possible. if not found, or wrong type, empty is returned. From a4cdbc429d15f0ea8e0101341ef7720a63c3d962 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:51:29 -0500 Subject: [PATCH 25/72] CE-936 - Add method inferNameFromBackendName --- .../core/instances/QInstanceEnricher.java | 44 +++++++++++++++++++ .../core/instances/QInstanceEnricherTest.java | 17 +++++++ 2 files changed, 61 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 232ade6c..45d249ed 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -1030,6 +1030,50 @@ public class QInstanceEnricher + /******************************************************************************* + ** Do a default mapping from an underscore_style field name to a camelCase name. + ** + ** Examples: + **
    + **
  • word_another_word_more_words -> wordAnotherWordMoreWords
    • + **
  • l_ul_ul_ul -> lUlUlUl
    • + **
  • tla_first -> tlaFirst
    • + **
  • word_then_tla_in_middle -> wordThenTlaInMiddle
    • + **
  • end_with_tla -> endWithTla
    • + **
  • tla_and_another_tla -> tlaAndAnotherTla
    • + **
  • ALL_CAPS -> allCaps
    • + **
+ *******************************************************************************/ + public static String inferNameFromBackendName(String backendName) + { + StringBuilder rs = new StringBuilder(); + + //////////////////////////////////////////////////////////////////////////////////////// + // build a list of words in the name, then join them with _ and lower-case the result // + //////////////////////////////////////////////////////////////////////////////////////// + String[] words = backendName.toLowerCase(Locale.ROOT).split("_"); + for(int i = 0; i < words.length; i++) + { + String word = words[i]; + if(i == 0) + { + rs.append(word); + } + else + { + rs.append(word.substring(0, 1).toUpperCase()); + if(word.length() > 1) + { + rs.append(word.substring(1)); + } + } + } + + return (rs.toString()); + } + + + /******************************************************************************* ** If a app didn't have any sections, generate "sensible defaults" *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java index 53e9ec96..a4ab55c9 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java @@ -232,6 +232,23 @@ class QInstanceEnricherTest extends BaseTest } + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testInferNameFromBackendName() + { + assertEquals("id", QInstanceEnricher.inferNameFromBackendName("id")); + assertEquals("wordAnotherWordMoreWords", QInstanceEnricher.inferNameFromBackendName("word_another_word_more_words")); + assertEquals("lUlUlUl", QInstanceEnricher.inferNameFromBackendName("l_ul_ul_ul")); + assertEquals("tlaFirst", QInstanceEnricher.inferNameFromBackendName("tla_first")); + assertEquals("wordThenTlaInMiddle", QInstanceEnricher.inferNameFromBackendName("word_then_tla_in_middle")); + assertEquals("endWithTla", QInstanceEnricher.inferNameFromBackendName("end_with_tla")); + assertEquals("tlaAndAnotherTla", QInstanceEnricher.inferNameFromBackendName("tla_and_another_tla")); + assertEquals("allCaps", QInstanceEnricher.inferNameFromBackendName("ALL_CAPS")); + } + + /******************************************************************************* ** From dabaafa4829e33a0589e0ae0cd58291f337f2186 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:53:13 -0500 Subject: [PATCH 26/72] CE-936 - Update setBlobValuesToDownloadUrls to only do this if the file has a fileDownload adornment --- .../qqq/backend/core/actions/values/QValueFormatter.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index 5b176bbc..726ceea4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -490,6 +490,13 @@ public class QValueFormatter { adornmentValues = fileDownloadAdornment.get().getValues(); } + else + { + /////////////////////////////////////////////////////// + // don't change blobs unless they are file-downloads // + /////////////////////////////////////////////////////// + continue; + } String fileNameField = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD)); String fileNameFormat = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT)); From 7155180a7619d0df23567cebfce774afa9cb261f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 11:55:14 -0500 Subject: [PATCH 27/72] CE-936 - Checkpoint on Quartz scheduler implementation. - Add QSchedulerMetaData as new type of top-level meta data - Move existing ScheduleManager to be SimpleScheduler, an instance of new QSchedulerInterface - Update QuartzScheduler to implement new QSchedulerInterface, plus: -- support cron schedules -- handle parallel variant jobs -- handle automations & sqs pollers --- .../core/instances/QInstanceValidator.java | 60 ++- .../core/model/metadata/QInstance.java | 54 +++ .../scheduleing/QScheduleMetaData.java | 109 +++++ .../scheduleing/QSchedulerMetaData.java | 131 ++++++ .../quartz/QuartzSchedulerMetaData.java | 120 +++++ .../simple/SimpleSchedulerMetaData.java | 76 ++++ .../core/scheduler/QScheduleManager.java | 278 ++++++++++++ .../core/scheduler/QSchedulerInterface.java | 84 ++++ .../core/scheduler/SchedulerUtils.java | 13 +- .../scheduler/quartz/QuartzRunProcessJob.java | 28 +- .../scheduler/quartz/QuartzScheduler.java | 413 ++++++++++++------ .../scheduler/quartz/QuartzSqsPollerJob.java | 79 ++++ .../quartz/QuartzTableAutomationsJob.java | 77 ++++ .../QuartzJobDetailsPostQueryCustomizer.java | 71 +++ .../SimpleScheduler.java} | 230 ++++------ .../StandardScheduledExecutor.java | 4 +- .../StandardScheduledExecutorTest.java | 2 +- .../instances/QInstanceValidatorTest.java | 123 ++++++ .../scheduler/quartz/QuartzSchedulerTest.java | 63 +++ .../SimpleSchedulerTest.java} | 27 +- .../qqq/backend/core/utils/TestUtils.java | 27 +- 21 files changed, 1770 insertions(+), 299 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDetailsPostQueryCustomizer.java rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/{ScheduleManager.java => simple/SimpleScheduler.java} (63%) rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/{ => simple}/StandardScheduledExecutor.java (98%) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java rename qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/{ScheduleManagerTest.java => simple/SimpleSchedulerTest.java} (85%) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 96661697..4582bbec 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -25,7 +25,9 @@ package com.kingsrook.qqq.backend.core.instances; import java.io.Serializable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Modifier; +import java.text.ParseException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -33,6 +35,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TimeZone; import java.util.function.Supplier; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; @@ -93,6 +96,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda; +import org.quartz.CronExpression; /******************************************************************************* @@ -342,6 +346,11 @@ public class QInstanceValidator assertCondition(StringUtils.hasContent(sqsQueueProvider.getSecretKey()), "Missing secretKey for SQSQueueProvider: " + name); assertCondition(StringUtils.hasContent(sqsQueueProvider.getBaseURL()), "Missing baseURL for SQSQueueProvider: " + name); assertCondition(StringUtils.hasContent(sqsQueueProvider.getRegion()), "Missing region for SQSQueueProvider: " + name); + + if(assertCondition(sqsQueueProvider.getSchedule() != null, "Missing schedule for SQSQueueProvider: " + name)) + { + validateScheduleMetaData(sqsQueueProvider.getSchedule(), qInstance, "SQSQueueProvider " + name + ", schedule: "); + } } }); } @@ -392,6 +401,11 @@ public class QInstanceValidator { assertCondition(Objects.equals(name, automationProvider.getName()), "Inconsistent naming for automationProvider: " + name + "/" + automationProvider.getName() + "."); assertCondition(automationProvider.getType() != null, "Missing type for automationProvider: " + name); + + if(assertCondition(automationProvider.getSchedule() != null, "Missing schedule for automationProvider: " + name)) + { + validateScheduleMetaData(automationProvider.getSchedule(), qInstance, "automationProvider " + name + ", schedule: "); + } }); } } @@ -1316,7 +1330,7 @@ public class QInstanceValidator if(process.getSchedule() != null) { QScheduleMetaData schedule = process.getSchedule(); - assertCondition(schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null, "Either repeat millis or repeat seconds must be set on schedule in process " + processName); + validateScheduleMetaData(schedule, qInstance, "Process " + processName + ", schedule: "); if(schedule.getVariantBackend() != null) { @@ -1336,6 +1350,50 @@ public class QInstanceValidator + /******************************************************************************* + ** + *******************************************************************************/ + private void validateScheduleMetaData(QScheduleMetaData schedule, QInstance qInstance, String prefix) + { + boolean isRepeat = schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null; + boolean isCron = StringUtils.hasContent(schedule.getCronExpression()); + assertCondition(isRepeat || isCron, prefix + " either repeatMillis or repeatSeconds or cronExpression must be set"); + assertCondition(!(isRepeat && isCron), prefix + " both a repeat time and cronExpression may not be set"); + + if(isCron) + { + boolean hasDelay = schedule.getInitialDelayMillis() != null || schedule.getInitialDelaySeconds() != null; + assertCondition(!hasDelay, prefix + " a cron schedule may not have an initial delay"); + + try + { + CronExpression.validateExpression(schedule.getCronExpression()); + } + catch(ParseException pe) + { + errors.add(prefix + " invalid cron expression: " + pe.getMessage()); + } + + if(assertCondition(StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a cron schedule must specify a cronTimeZoneId")) + { + String[] availableIDs = TimeZone.getAvailableIDs(); + Optional first = Arrays.stream(availableIDs).filter(id -> id.equals(schedule.getCronTimeZoneId())).findFirst(); + assertCondition(first.isPresent(), prefix + " unrecognized cronTimeZoneId: " + schedule.getCronTimeZoneId()); + } + } + else + { + assertCondition(!StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a non-cron schedule must not specify a cronTimeZoneId"); + } + + if(assertCondition(StringUtils.hasContent(schedule.getSchedulerName()), prefix + " is missing a scheduler name")) + { + assertCondition(qInstance.getScheduler(schedule.getSchedulerName()) != null, prefix + " is referencing an unknown scheduler name: " + schedule.getSchedulerName()); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 115fcccc..5a5b4ac3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -53,6 +53,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -90,6 +91,7 @@ public class QInstance private Map widgets = new LinkedHashMap<>(); private Map queueProviders = new LinkedHashMap<>(); private Map queues = new LinkedHashMap<>(); + private Map schedulers = new LinkedHashMap<>(); private Map supplementalMetaData = new LinkedHashMap<>(); @@ -1224,4 +1226,56 @@ public class QInstance metaData.addSelfToInstance(this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addScheduler(QSchedulerMetaData scheduler) + { + String name = scheduler.getName(); + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a scheduler without a name.")); + } + if(this.schedulers.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second scheduler with name: " + name)); + } + this.schedulers.put(name, scheduler); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QSchedulerMetaData getScheduler(String name) + { + return (this.schedulers.get(name)); + } + + + + /******************************************************************************* + ** Getter for schedulers + ** + *******************************************************************************/ + public Map getSchedulers() + { + return schedulers; + } + + + + /******************************************************************************* + ** Setter for schedulers + ** + *******************************************************************************/ + public void setSchedulers(Map schedulers) + { + this.schedulers = schedulers; + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java index ece9019a..c7373f1a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java @@ -22,6 +22,9 @@ package com.kingsrook.qqq.backend.core.model.metadata.scheduleing; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + /******************************************************************************* ** Meta-data to define scheduled actions within QQQ. ** @@ -37,16 +40,29 @@ public class QScheduleMetaData {PARALLEL, SERIAL} + private String schedulerName; private Integer repeatSeconds; private Integer repeatMillis; private Integer initialDelaySeconds; private Integer initialDelayMillis; + private String cronExpression; + private String cronTimeZoneId; + private RunStrategy variantRunStrategy; private String variantBackend; + /******************************************************************************* + ** + *******************************************************************************/ + public boolean isCron() + { + return StringUtils.hasContent(cronExpression); + } + + /******************************************************************************* ** Getter for repeatSeconds @@ -244,4 +260,97 @@ public class QScheduleMetaData return (this); } + + /******************************************************************************* + ** Getter for cronExpression + *******************************************************************************/ + public String getCronExpression() + { + return (this.cronExpression); + } + + + + /******************************************************************************* + ** Setter for cronExpression + *******************************************************************************/ + public void setCronExpression(String cronExpression) + { + this.cronExpression = cronExpression; + } + + + + /******************************************************************************* + ** Fluent setter for cronExpression + *******************************************************************************/ + public QScheduleMetaData withCronExpression(String cronExpression) + { + this.cronExpression = cronExpression; + return (this); + } + + + + /******************************************************************************* + ** Getter for cronTimeZoneId + *******************************************************************************/ + public String getCronTimeZoneId() + { + return (this.cronTimeZoneId); + } + + + + /******************************************************************************* + ** Setter for cronTimeZoneId + *******************************************************************************/ + public void setCronTimeZoneId(String cronTimeZoneId) + { + this.cronTimeZoneId = cronTimeZoneId; + } + + + + /******************************************************************************* + ** Fluent setter for cronTimeZoneId + *******************************************************************************/ + public QScheduleMetaData withCronTimeZoneId(String cronTimeZoneId) + { + this.cronTimeZoneId = cronTimeZoneId; + return (this); + } + + + + /******************************************************************************* + ** Getter for schedulerName + *******************************************************************************/ + public String getSchedulerName() + { + return (this.schedulerName); + } + + + + /******************************************************************************* + ** Setter for schedulerName + *******************************************************************************/ + public void setSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + } + + + + /******************************************************************************* + ** Fluent setter for schedulerName + *******************************************************************************/ + public QScheduleMetaData withSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java new file mode 100644 index 00000000..90fb8f92 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java @@ -0,0 +1,131 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.scheduleing; + + +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class QSchedulerMetaData implements TopLevelMetaDataInterface +{ + private String name; + private String type; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public boolean supportsCronSchedules() + { + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public abstract QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier systemSessionSupplier) throws QException; + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public QSchedulerMetaData withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for type + *******************************************************************************/ + public String getType() + { + return (this.type); + } + + + + /******************************************************************************* + ** Setter for type + *******************************************************************************/ + public void setType(String type) + { + this.type = type; + } + + + + /******************************************************************************* + ** Fluent setter for type + *******************************************************************************/ + public QSchedulerMetaData withType(String type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void addSelfToInstance(QInstance qInstance) + { + qInstance.addScheduler(this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java new file mode 100644 index 00000000..12ef0361 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java @@ -0,0 +1,120 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz; + + +import java.util.Properties; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuartzSchedulerMetaData extends QSchedulerMetaData +{ + private static final QLogger LOG = QLogger.getLogger(QuartzSchedulerMetaData.class); + + public static final String TYPE = "quartz"; + + private Properties properties; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public QuartzSchedulerMetaData() + { + setType(TYPE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean supportsCronSchedules() + { + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier systemSessionSupplier) throws QException + { + try + { + QuartzScheduler quartzScheduler = QuartzScheduler.initInstance(qInstance, getName(), getProperties(), systemSessionSupplier); + return (quartzScheduler); + } + catch(Exception e) + { + LOG.error("Error initializing quartz scheduler", e); + throw (new QException("Error initializing quartz scheduler", e)); + } + } + + + + /******************************************************************************* + ** Getter for properties + *******************************************************************************/ + public Properties getProperties() + { + return (this.properties); + } + + + + /******************************************************************************* + ** Setter for properties + *******************************************************************************/ + public void setProperties(Properties properties) + { + this.properties = properties; + } + + + + /******************************************************************************* + ** Fluent setter for properties + *******************************************************************************/ + public QuartzSchedulerMetaData withProperties(Properties properties) + { + this.properties = properties; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java new file mode 100644 index 00000000..92ef2c99 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java @@ -0,0 +1,76 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple; + + +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.simple.SimpleScheduler; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SimpleSchedulerMetaData extends QSchedulerMetaData +{ + public static final String TYPE = "simple"; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SimpleSchedulerMetaData() + { + setType(TYPE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean supportsCronSchedules() + { + return (false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QSchedulerInterface initSchedulerInstance(QInstance qInstance, Supplier systemSessionSupplier) + { + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); + simpleScheduler.setSessionSupplier(systemSessionSupplier); + simpleScheduler.setSchedulerName(getName()); + return simpleScheduler; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java new file mode 100644 index 00000000..34302e23 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -0,0 +1,278 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.context.CapturedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; +import com.kingsrook.qqq.backend.core.logging.LogPair; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** QQQ service to manage scheduled jobs, using 1 or more Schedulers - implementations + ** of the QSchedulerInterface + *******************************************************************************/ +public class QScheduleManager +{ + private static final QLogger LOG = QLogger.getLogger(QScheduleManager.class); + + private static QScheduleManager qScheduleManager = null; + private final QInstance qInstance; + private final Supplier systemUserSessionSupplier; + + private Map schedulers = new HashMap<>(); + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private QScheduleManager(QInstance qInstance, Supplier systemUserSessionSupplier) + { + this.qInstance = qInstance; + this.systemUserSessionSupplier = systemUserSessionSupplier; + } + + + + /******************************************************************************* + ** Singleton initiator - e.g., must be called to initially initialize the singleton + ** before anyone else calls getInstance (they'll get an error if they call that first). + *******************************************************************************/ + public static QScheduleManager initInstance(QInstance qInstance, Supplier systemUserSessionSupplier) + { + if(qScheduleManager == null) + { + qScheduleManager = new QScheduleManager(qInstance, systemUserSessionSupplier); + } + return (qScheduleManager); + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static QScheduleManager getInstance() + { + if(qScheduleManager == null) + { + throw (new IllegalStateException("QScheduleManager singleton has not been init'ed (call initInstance).")); + } + return (qScheduleManager); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void start() throws QException + { + if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) + { + LOG.info("Not starting ScheduleManager per settings."); + return; + } + + ///////////////////////////////////////////////////////// + // initialize the scheduler(s) we're configured to use // + ///////////////////////////////////////////////////////// + for(QSchedulerMetaData schedulerMetaData : qInstance.getSchedulers().values()) + { + QSchedulerInterface scheduler = schedulerMetaData.initSchedulerInstance(qInstance, systemUserSessionSupplier); + schedulers.put(schedulerMetaData.getName(), scheduler); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // ensure that everything which should be scheduled is scheduled, in the appropriate scheduler // + ///////////////////////////////////////////////////////////////////////////////////////////////// + QContext.withTemporaryContext(new CapturedContext(qInstance, systemUserSessionSupplier.get()), () -> setupSchedules()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupSchedules() + { + ///////////////////////////////////////////////////////// + // let the schedulers know we're starting this process // + ///////////////////////////////////////////////////////// + schedulers.values().forEach(s -> s.startOfSetupSchedules()); + + ////////////////////////////////// + // schedule all queue providers // + ////////////////////////////////// + for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values()) + { + setupQueueProvider(queueProvider); + } + + /////////////////////////////////////// + // schedule all automation providers // + /////////////////////////////////////// + for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values()) + { + setupAutomationProviderPerTable(automationProvider); + } + + ///////////////////////////////////////// + // schedule all processes that need it // + ///////////////////////////////////////// + for(QProcessMetaData process : qInstance.getProcesses().values()) + { + if(process.getSchedule() != null) + { + QScheduleMetaData scheduleMetaData = process.getSchedule(); + if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) + { + /////////////////////////////////////////////// + // if no variants, or variant is serial mode // + /////////////////////////////////////////////// + setupProcess(process, null); + } + else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // if this a "parallel", which for example means we want to have a thread for each backend variant // + // running at the same time, get the variant records and schedule each separately // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); + for(QRecord qRecord : CollectionUtils.nonNullList(SchedulerUtils.getBackendVariantFilteredRecords(process))) + { + try + { + setupProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); + } + catch(Exception e) + { + LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); + } + } + } + else + { + LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided."); + } + } + } + + ///////////////////////////////////////////////////////////// + // todo - read dynamic schedules and schedule those things // + // e.g., user-scheduled processes, reports // + ///////////////////////////////////////////////////////////// + + ////////////////////////////////////////////////////////// + // let the schedulers know we're done with this process // + ////////////////////////////////////////////////////////// + schedulers.values().forEach(s -> s.endOfSetupSchedules()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupProcess(QProcessMetaData process, Map backendVariantData) + { + QSchedulerInterface scheduler = getScheduler(process.getSchedule().getSchedulerName()); + scheduler.setupProcess(process, backendVariantData, SchedulerUtils.allowedToStart(process)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupQueueProvider(QQueueProviderMetaData queueProvider) + { + switch(queueProvider.getType()) + { + case SQS: + setupSqsProvider((SQSQueueProviderMetaData) queueProvider); + break; + default: + throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupSqsProvider(SQSQueueProviderMetaData queueProvider) + { + QSchedulerInterface scheduler = getScheduler(queueProvider.getSchedule().getSchedulerName()); + scheduler.setupSqsProvider(queueProvider, SchedulerUtils.allowedToStart(queueProvider)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider) + { + QSchedulerInterface scheduler = getScheduler(automationProvider.getSchedule().getSchedulerName()); + scheduler.setupAutomationProviderPerTable(automationProvider, SchedulerUtils.allowedToStart(automationProvider)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QSchedulerInterface getScheduler(String schedulerName) + { + QSchedulerInterface scheduler = schedulers.get(schedulerName); + if(scheduler == null) + { + throw new NotImplementedException("default scheduler..."); + } + + return (scheduler); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java new file mode 100644 index 00000000..62404fb0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java @@ -0,0 +1,84 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.io.Serializable; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface QSchedulerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + void setupSqsProvider(SQSQueueProviderMetaData queueProvider, boolean allowedToStart); + + /******************************************************************************* + ** + *******************************************************************************/ + void stopAsync(); + + /******************************************************************************* + ** + *******************************************************************************/ + void stop(); + + /******************************************************************************* + ** + *******************************************************************************/ + void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStart); + + /******************************************************************************* + ** + *******************************************************************************/ + void setupProcess(QProcessMetaData process, Map backendVariantData, boolean allowedToStart); + + /******************************************************************************* + ** + *******************************************************************************/ + void start(); + + /******************************************************************************* + ** let the scheduler know when the schedule manager is at the start of setting up schedules. + *******************************************************************************/ + default void startOfSetupSchedules() + { + + } + + /******************************************************************************* + ** let the scheduler know when the schedule manager is at the end of setting up schedules. + *******************************************************************************/ + default void endOfSetupSchedules() + { + + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java index 76f86394..e44c124d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -48,7 +49,7 @@ import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; /******************************************************************************* - ** + ** Utility methods used by various schedulers. *******************************************************************************/ public class SchedulerUtils { @@ -56,6 +57,16 @@ public class SchedulerUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static boolean allowedToStart(TopLevelMetaDataInterface metaDataObject) + { + return (allowedToStart(metaDataObject.getName())); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java index a5bb27f3..3cfde45f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java @@ -27,6 +27,7 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; @@ -57,16 +58,31 @@ public class QuartzRunProcessJob implements Job JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); String processName = jobDataMap.getString("processName"); - /////////////////////////////////// - // todo - variants from job data // - /////////////////////////////////// + /////////////////////////////////////// + // get the process from the instance // + /////////////////////////////////////// + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + QProcessMetaData process = qInstance.getProcess(processName); + if(process == null) + { + LOG.warn("Could not find scheduled process in QInstance", logPair("processName", processName)); + return; + } + + /////////////////////////////////////////////// + // if the job has variant data, get it ready // + /////////////////////////////////////////////// Map backendVariantData = null; + if(jobExecutionContext.getMergedJobDataMap().containsKey("backendVariantData")) + { + backendVariantData = (Map) jobExecutionContext.getMergedJobDataMap().get("backendVariantData"); + } + ///////////// + // run it. // + ///////////// LOG.debug("Running quartz process", logPair("processName", processName)); - - QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); SchedulerUtils.runProcess(qInstance, QuartzScheduler.getInstance().getSessionSupplier(), qInstance.getProcess(processName), backendVariantData); - } finally { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java index 5a8385f6..da9fe9ca 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java @@ -23,20 +23,38 @@ package com.kingsrook.qqq.backend.core.scheduler.quartz; import java.io.Serializable; +import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.Date; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.TimeZone; import java.util.function.Supplier; -import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.AnyKey; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; +import org.quartz.CronExpression; +import org.quartz.CronScheduleBuilder; +import org.quartz.Job; import org.quartz.JobBuilder; import org.quartz.JobDetail; import org.quartz.JobKey; +import org.quartz.ScheduleBuilder; import org.quartz.Scheduler; import org.quartz.SchedulerException; import org.quartz.SimpleScheduleBuilder; @@ -49,41 +67,60 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* - ** + ** Singleton to provide access between QQQ and the quartz Scheduler system. *******************************************************************************/ -public class QuartzScheduler +public class QuartzScheduler implements QSchedulerInterface { private static final QLogger LOG = QLogger.getLogger(QuartzScheduler.class); private static QuartzScheduler quartzScheduler = null; private final QInstance qInstance; + private String schedulerName; + private Properties quartzProperties; private Supplier sessionSupplier; private Scheduler scheduler; + private Memoization> jobGroupNamesMemoization = new Memoization>() + .withTimeout(Duration.of(5, ChronoUnit.SECONDS)); + + private Memoization> jobKeyNamesMemoization = new Memoization>() + .withTimeout(Duration.of(5, ChronoUnit.SECONDS)); + /******************************************************************************* ** Constructor ** *******************************************************************************/ - private QuartzScheduler(QInstance qInstance, Supplier sessionSupplier) + private QuartzScheduler(QInstance qInstance, String schedulerName, Properties quartzProperties, Supplier sessionSupplier) { this.qInstance = qInstance; + this.schedulerName = schedulerName; + this.quartzProperties = quartzProperties; this.sessionSupplier = sessionSupplier; } /******************************************************************************* - ** Singleton initiator... + ** Singleton initiator - e.g., must be called to initially initialize the singleton + ** before anyone else calls getInstance (they'll get an error if they call that first). *******************************************************************************/ - public static QuartzScheduler initInstance(QInstance qInstance, Supplier sessionSupplier) + public static QuartzScheduler initInstance(QInstance qInstance, String schedulerName, Properties quartzProperties, Supplier sessionSupplier) throws SchedulerException { if(quartzScheduler == null) { - quartzScheduler = new QuartzScheduler(qInstance, sessionSupplier); + quartzScheduler = new QuartzScheduler(qInstance, schedulerName, quartzProperties, sessionSupplier); + + /////////////////////////////////////////////////////////// + // Grab the Scheduler instance from the Factory // + // initialize it with the properties we took in as input // + /////////////////////////////////////////////////////////// + StdSchedulerFactory schedulerFactory = new StdSchedulerFactory(); + schedulerFactory.initialize(quartzProperties); + quartzScheduler.scheduler = schedulerFactory.getScheduler(); } return (quartzScheduler); } @@ -97,7 +134,7 @@ public class QuartzScheduler { if(quartzScheduler == null) { - throw (new IllegalStateException("QuartzScheduler singleton has not been init'ed.")); + throw (new IllegalStateException("QuartzScheduler singleton has not been init'ed (call initInstance).")); } return (quartzScheduler); } @@ -111,30 +148,6 @@ public class QuartzScheduler { try { - // Properties properties = new Properties(); - // properties.put(""); - - ////////////////////////////////////////////////// - // Grab the Scheduler instance from the Factory // - ////////////////////////////////////////////////// - StdSchedulerFactory schedulerFactory = new StdSchedulerFactory(); - // schedulerFactory.initialize(properties); - this.scheduler = schedulerFactory.getScheduler(); - - //////////////////////////////////////// - // todo - do we get our own property? // - //////////////////////////////////////// - if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) - { - LOG.info("Not starting QuartzScheduler per settings."); - return; - } - - ///////////////////////////////////////////// - // make sure all of our jobs are scheduled // - ///////////////////////////////////////////// - scheduleAllJobs(); - ////////////////////// // and start it off // ////////////////////// @@ -151,20 +164,166 @@ public class QuartzScheduler /******************************************************************************* ** *******************************************************************************/ - private void scheduleAllJobs() + @Override + public void stop() { - for(QProcessMetaData process : qInstance.getProcesses().values()) + try { - if(process.getSchedule() != null && SchedulerUtils.allowedToStart(process.getName())) + scheduler.shutdown(true); + } + catch(SchedulerException e) + { + LOG.error("Error shutting down (stopping) quartz scheduler", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stopAsync() + { + try + { + scheduler.shutdown(false); + } + catch(SchedulerException e) + { + LOG.error("Error shutting down (stopping) quartz scheduler", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setupProcess(QProcessMetaData process, Map backendVariantData, boolean allowedToStart) + { + ///////////////////////// + // set up job data map // + ///////////////////////// + Map jobData = new HashMap<>(); + jobData.put("processName", process.getName()); + + if(backendVariantData != null) + { + jobData.put("backendVariantData", backendVariantData); + } + + scheduleJob(process.getName(), "processes", QuartzRunProcessJob.class, jobData, process.getSchedule(), allowedToStart); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean scheduleJob(String jobName, String groupName, Class jobClass, Map jobData, QScheduleMetaData scheduleMetaData, boolean allowedToStart) + { + try + { + ///////////////////////// + // Define job instance // + ///////////////////////// + JobKey jobKey = new JobKey(jobName, groupName); + JobDetail jobDetail = JobBuilder.newJob(jobClass) + .withIdentity(jobKey) + .storeDurably() + .requestRecovery() + .build(); + + jobDetail.getJobDataMap().putAll(jobData); + + ///////////////////////////////////////////////////////// + // map the qqq schedule meta data to a quartz schedule // + ///////////////////////////////////////////////////////// + ScheduleBuilder scheduleBuilder; + if(scheduleMetaData.isCron()) { - if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) - { - scheduleProcess(process, null); - } - else - { - LOG.error("Not yet know how to schedule parallel variant jobs"); - } + CronExpression cronExpression = new CronExpression(scheduleMetaData.getCronExpression()); + cronExpression.setTimeZone(TimeZone.getTimeZone(scheduleMetaData.getCronTimeZoneId())); + scheduleBuilder = CronScheduleBuilder.cronSchedule(cronExpression); + } + else + { + long intervalMillis = Objects.requireNonNullElse(scheduleMetaData.getRepeatMillis(), scheduleMetaData.getRepeatSeconds() * 1000); + scheduleBuilder = SimpleScheduleBuilder.simpleSchedule() + .withIntervalInMilliseconds(intervalMillis) + .repeatForever(); + } + + Date startAt = new Date(); + if(scheduleMetaData.getInitialDelayMillis() != null) + { + startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelayMillis()); + } + else if(scheduleMetaData.getInitialDelaySeconds() != null) + { + startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelaySeconds() * 1000); + } + + /////////////////////////////////////// + // Define a Trigger for the schedule // + /////////////////////////////////////// + Trigger trigger = TriggerBuilder.newTrigger() + .withIdentity(new TriggerKey(jobName, groupName)) + .forJob(jobKey) + .withSchedule(scheduleBuilder) + .startAt(startAt) + .build(); + + /////////////////////////////////////// + // Schedule the job with the trigger // + /////////////////////////////////////// + addOrReplaceJobAndTrigger(jobKey, jobDetail, trigger); + + ////////////////////////////////////////////////////////// + // either pause or resume, based on if allowed to start // + ////////////////////////////////////////////////////////// + if(!allowedToStart) + { + pauseJob(jobKey.getName(), jobKey.getGroup()); + } + else + { + resumeJob(jobKey.getName(), jobKey.getGroup()); + } + + return (true); + } + catch(Exception e) + { + LOG.warn("Error scheduling job", e, logPair("name", jobName), logPair("group", groupName)); + return (false); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setupSqsProvider(SQSQueueProviderMetaData sqsQueueProvider, boolean allowedToStartProvider) + { + for(QQueueMetaData queue : qInstance.getQueues().values()) + { + boolean allowedToStart = allowedToStartProvider && SchedulerUtils.allowedToStart(queue.getName()); + + if(sqsQueueProvider.getName().equals(queue.getProviderName())) + { + ///////////////////////// + // set up job data map // + ///////////////////////// + Map jobData = new HashMap<>(); + jobData.put("queueProviderName", sqsQueueProvider.getName()); + jobData.put("queueName", queue.getName()); + + scheduleJob(queue.getName(), "sqsQueue", QuartzSqsPollerJob.class, jobData, sqsQueueProvider.getSchedule(), allowedToStart); } } } @@ -174,85 +333,75 @@ public class QuartzScheduler /******************************************************************************* ** *******************************************************************************/ - private void scheduleProcess(QProcessMetaData process, Map backendVariantData) + @Override + public void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStartProvider) { - try + /////////////////////////////////////////////////////////////////////////////////// + // ask the PollingAutomationPerTableRunner how many threads of itself need setup // + // then start a scheduled executor foreach one // + /////////////////////////////////////////////////////////////////////////////////// + List tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName()); + for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions) { - QScheduleMetaData scheduleMetaData = process.getSchedule(); - long intervalMillis = Objects.requireNonNullElse(scheduleMetaData.getRepeatMillis(), scheduleMetaData.getRepeatSeconds() * 1000); - - Date startAt = new Date(); - if(scheduleMetaData.getInitialDelayMillis() != null) - { - startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelayMillis()); - } - if(scheduleMetaData.getInitialDelaySeconds() != null) - { - startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelaySeconds() * 1000); - } + boolean allowedToStart = allowedToStartProvider && SchedulerUtils.allowedToStart(tableAction.tableName()); ///////////////////////// - // Define job instance // + // set up job data map // ///////////////////////// - JobKey jobKey = new JobKey(process.getName(), "processes"); - JobDetail jobDetail = JobBuilder.newJob(QuartzRunProcessJob.class) - .withIdentity(jobKey) - .storeDurably() - .requestRecovery() - .build(); + Map jobData = new HashMap<>(); + jobData.put("automationProviderName", automationProvider.getName()); + jobData.put("tableName", tableAction.tableName()); + jobData.put("automationStatus", tableAction.status().toString()); - jobDetail.getJobDataMap().put("processName", process.getName()); - - /////////////////////////////////////// - // Define a Trigger for the schedule // - /////////////////////////////////////// - Trigger trigger = TriggerBuilder.newTrigger() - .withIdentity(new TriggerKey(process.getName(), "processes")) - .forJob(jobKey) - .withSchedule(SimpleScheduleBuilder.simpleSchedule() - .withIntervalInMilliseconds(intervalMillis) - .repeatForever()) - .startAt(startAt) - .build(); - - /////////////////////////////////////// - // Schedule the job with the trigger // - /////////////////////////////////////// - boolean isJobAlreadyScheduled = isJobAlreadyScheduled(jobKey); - if(isJobAlreadyScheduled) - { - this.scheduler.addJob(jobDetail, true); - this.scheduler.rescheduleJob(trigger.getKey(), trigger); - LOG.info("Re-scheduled process: " + process.getName()); - } - else - { - this.scheduler.scheduleJob(jobDetail, trigger); - LOG.info("Scheduled new process: " + process.getName()); - } - - } - catch(Exception e) - { - LOG.warn("Error scheduling process", e, logPair("processName", process.getName())); + scheduleJob(tableAction.tableName() + "." + tableAction.status(), "tableAutomations", QuartzTableAutomationsJob.class, jobData, automationProvider.getSchedule(), allowedToStart); } } /******************************************************************************* - ** todo - probably rewrite this to not re-query quartz each time + ** + *******************************************************************************/ + private void addOrReplaceJobAndTrigger(JobKey jobKey, JobDetail jobDetail, Trigger trigger) throws SchedulerException + { + boolean isJobAlreadyScheduled = isJobAlreadyScheduled(jobKey); + if(isJobAlreadyScheduled) + { + this.scheduler.addJob(jobDetail, true); + this.scheduler.rescheduleJob(trigger.getKey(), trigger); + LOG.info("Re-scheduled job: " + jobKey); + } + else + { + this.scheduler.scheduleJob(jobDetail, trigger); + LOG.info("Scheduled new job: " + jobKey); + } + } + + + + /******************************************************************************* + ** *******************************************************************************/ private boolean isJobAlreadyScheduled(JobKey jobKey) throws SchedulerException { - for(String group : scheduler.getJobGroupNames()) + Optional> jobGroupNames = jobGroupNamesMemoization.getResult(AnyKey.getInstance(), (x) -> scheduler.getJobGroupNames()); + if(jobGroupNames.isEmpty()) { - for(JobKey testJobKey : scheduler.getJobKeys(GroupMatcher.groupEquals(group))) + throw (new SchedulerException("Error getting job group names")); + } + + for(String group : jobGroupNames.get()) + { + Optional> jobKeys = jobKeyNamesMemoization.getResult(group, (x) -> scheduler.getJobKeys(GroupMatcher.groupEquals(group))); + if(jobKeys.isEmpty()) { - if(testJobKey.equals(jobKey)) - { - return (true); - } + throw (new SchedulerException("Error getting job keys")); + } + + if(jobKeys.get().contains(jobKey)) + { + return (true); } } @@ -261,45 +410,33 @@ public class QuartzScheduler - /* - private void todo() throws SchedulerException + /******************************************************************************* + ** + *******************************************************************************/ + private boolean deleteJob(JobKey jobKey) { - // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/ListJobs.html - // Listing all Jobs in the scheduler - for(String group : scheduler.getJobGroupNames()) + try { - for(JobKey jobKey : scheduler.getJobKeys(GroupMatcher.groupEquals(group))) + ///////////////////////////////////////////////////////////////////////////////////////////// + // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UnscheduleJob.html // + // Deleting a Job and Unscheduling All of Its Triggers // + ///////////////////////////////////////////////////////////////////////////////////////////// + if(isJobAlreadyScheduled(jobKey)) { - System.out.println("Found job identified by: " + jobKey); + return scheduler.deleteJob(jobKey); } + + ///////////////////////////////////////// + // return true to indicate, we're good // + ///////////////////////////////////////// + return (true); + } + catch(Exception e) + { + LOG.warn("Error deleting job", e, logPair("jobKey", jobKey)); + return false; } - - // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UpdateJob.html - // Update an existing job - // Add the new job to the scheduler, instructing it to "replace" - // the existing job with the given name and group (if any) - JobDetail jobDetail = JobBuilder.newJob(QuartzRunProcessJob.class) - .withIdentity("job1", "group1") - .build(); - // store, and set overwrite flag to 'true' - scheduler.addJob(jobDetail, true); - - // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UpdateTrigger.html - // Define a new Trigger - Trigger trigger = TriggerBuilder.newTrigger() - .withIdentity("newTrigger", "group1") - .startNow() - .build(); - - // tell the scheduler to remove the old trigger with the given key, and put the new one in its place - scheduler.rescheduleJob(new TriggerKey("oldTrigger", "group1"), trigger); - - // https://www.quartz-scheduler.org/documentation/quartz-2.3.0/cookbook/UnscheduleJob.html - // Deleting a Job and Unscheduling All of Its Triggers - scheduler.deleteJob(new JobKey("job1", "group1")); - } - */ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java new file mode 100644 index 00000000..7270d7fa --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java @@ -0,0 +1,79 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +@DisallowConcurrentExecution +public class QuartzSqsPollerJob implements Job +{ + private static final QLogger LOG = QLogger.getLogger(QuartzSqsPollerJob.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException + { + try + { + JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); + String queueProviderName = jobDataMap.getString("queueProviderName"); + String queueName = jobDataMap.getString("queueName"); + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + + SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); + sqsQueuePoller.setQueueProviderMetaData((SQSQueueProviderMetaData) qInstance.getQueueProvider(queueProviderName)); + sqsQueuePoller.setQueueMetaData(qInstance.getQueue(queueName)); + sqsQueuePoller.setQInstance(qInstance); + sqsQueuePoller.setSessionSupplier(QuartzScheduler.getInstance().getSessionSupplier()); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running quartz SQS Poller", logPair("queueName", queueName)); + sqsQueuePoller.run(); + } + finally + { + QContext.clear(); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java new file mode 100644 index 00000000..4f4f244b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +@DisallowConcurrentExecution +public class QuartzTableAutomationsJob implements Job +{ + private static final QLogger LOG = QLogger.getLogger(QuartzTableAutomationsJob.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException + { + try + { + JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); + String tableName = jobDataMap.getString("tableName"); + String automationProviderName = jobDataMap.getString("automationProviderName"); + AutomationStatus automationStatus = AutomationStatus.valueOf(jobDataMap.getString("automationStatus")); + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + + PollingAutomationPerTableRunner.TableActionsInterface tableAction = new PollingAutomationPerTableRunner.TableActions(tableName, automationStatus); + PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProviderName, QuartzScheduler.getInstance().getSessionSupplier(), tableAction); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running Table Automations", logPair("tableName", tableName), logPair("automationStatus", automationStatus)); + runner.run(); + } + finally + { + QContext.clear(); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDetailsPostQueryCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDetailsPostQueryCustomizer.java new file mode 100644 index 00000000..3c08d4ba --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDetailsPostQueryCustomizer.java @@ -0,0 +1,71 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.apache.commons.lang3.SerializationUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuartzJobDetailsPostQueryCustomizer extends AbstractPostQueryCustomizer +{ + private static final QLogger LOG = QLogger.getLogger(QuartzJobDetailsPostQueryCustomizer.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List apply(List records) + { + for(QRecord record : records) + { + if(record.getValue("jobData") != null) + { + try + { + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // this field has a blob of essentially a serialized map - so, deserialize that, then convert to JSON // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + byte[] value = record.getValueByteArray("jobData"); + Object deserialize = SerializationUtils.deserialize(value); + String json = JsonUtils.toJson(deserialize); + record.setValue("jobData", json); + } + catch(Exception e) + { + LOG.info("Error deserializing quartz job data", e); + } + } + } + return (records); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java similarity index 63% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java index cb7e0388..ed215833 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.scheduler; +package com.kingsrook.qqq.backend.core.scheduler.simple; import java.io.Serializable; @@ -30,22 +30,16 @@ import java.util.Objects; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; -import com.kingsrook.qqq.backend.core.logging.LogPair; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; /******************************************************************************* @@ -58,12 +52,13 @@ import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; ** ** All of these jobs run using a "system session" - as defined by the sessionSupplier. *******************************************************************************/ -public class ScheduleManager +public class SimpleScheduler implements QSchedulerInterface { - private static final QLogger LOG = QLogger.getLogger(ScheduleManager.class); + private static final QLogger LOG = QLogger.getLogger(SimpleScheduler.class); - private static ScheduleManager scheduleManager = null; + private static SimpleScheduler simpleScheduler = null; private final QInstance qInstance; + private String schedulerName; protected Supplier sessionSupplier; @@ -79,7 +74,7 @@ public class ScheduleManager /******************************************************************************* ** Singleton constructor *******************************************************************************/ - private ScheduleManager(QInstance qInstance) + private SimpleScheduler(QInstance qInstance) { this.qInstance = qInstance; } @@ -89,13 +84,13 @@ public class ScheduleManager /******************************************************************************* ** Singleton accessor *******************************************************************************/ - public static ScheduleManager getInstance(QInstance qInstance) + public static SimpleScheduler getInstance(QInstance qInstance) { - if(scheduleManager == null) + if(simpleScheduler == null) { - scheduleManager = new ScheduleManager(qInstance); + simpleScheduler = new SimpleScheduler(qInstance); } - return (scheduleManager); + return (simpleScheduler); } @@ -103,88 +98,56 @@ public class ScheduleManager /******************************************************************************* ** *******************************************************************************/ + @Override public void start() { - if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) + for(StandardScheduledExecutor executor : executors) + { + executor.start(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stopAsync() + { + for(StandardScheduledExecutor scheduledExecutor : executors) + { + scheduledExecutor.stopAsync(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void stop() + { + for(StandardScheduledExecutor scheduledExecutor : executors) + { + scheduledExecutor.stop(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStartProvider) + { + if(!allowedToStartProvider) { - LOG.info("Not starting ScheduleManager per settings."); return; } - boolean needToClearContext = false; - try - { - if(QContext.getQInstance() == null) - { - needToClearContext = true; - QContext.init(qInstance, sessionSupplier.get()); - } - - for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values()) - { - startQueueProvider(queueProvider); - } - - for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values()) - { - startAutomationProviderPerTable(automationProvider); - } - - for(QProcessMetaData process : qInstance.getProcesses().values()) - { - if(process.getSchedule() != null && SchedulerUtils.allowedToStart(process.getName())) - { - QScheduleMetaData scheduleMetaData = process.getSchedule(); - if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) - { - /////////////////////////////////////////////// - // if no variants, or variant is serial mode // - /////////////////////////////////////////////// - startProcess(process, null); - } - else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) - { - ///////////////////////////////////////////////////////////////////////////////////////////////////// - // if this a "parallel", which for example means we want to have a thread for each backend variant // - // running at the same time, get the variant records and schedule each separately // - ///////////////////////////////////////////////////////////////////////////////////////////////////// - QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); - for(QRecord qRecord : CollectionUtils.nonNullList(SchedulerUtils.getBackendVariantFilteredRecords(process))) - { - try - { - startProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); - } - catch(Exception e) - { - LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); - } - } - } - else - { - LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided."); - } - } - } - } - finally - { - if(needToClearContext) - { - QContext.clear(); - } - } - } - - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void startAutomationProviderPerTable(QAutomationProviderMetaData automationProvider) - { /////////////////////////////////////////////////////////////////////////////////// // ask the PollingAutomationPerTableRunner how many threads of itself need setup // // then start a scheduled executor foreach one // @@ -201,11 +164,6 @@ public class ScheduleManager executor.setName(runner.getName()); setScheduleInExecutor(schedule, executor); - if(!executor.start()) - { - LOG.warn("executor.start return false for: " + executor.getName()); - } - executors.add(executor); } } @@ -216,28 +174,14 @@ public class ScheduleManager /******************************************************************************* ** *******************************************************************************/ - private void startQueueProvider(QQueueProviderMetaData queueProvider) + @Override + public void setupSqsProvider(SQSQueueProviderMetaData queueProvider, boolean allowedToStartProvider) { - if(SchedulerUtils.allowedToStart(queueProvider.getName())) + if(!allowedToStartProvider) { - switch(queueProvider.getType()) - { - case SQS: - startSqsProvider((SQSQueueProviderMetaData) queueProvider); - break; - default: - throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType()); - } + return; } - } - - - /******************************************************************************* - ** - *******************************************************************************/ - private void startSqsProvider(SQSQueueProviderMetaData queueProvider) - { QInstance scheduleManagerQueueInstance = qInstance; Supplier scheduleManagerSessionSupplier = sessionSupplier; @@ -259,11 +203,6 @@ public class ScheduleManager executor.setName(queue.getName()); setScheduleInExecutor(schedule, executor); - if(!executor.start()) - { - LOG.warn("executor.start return false for: " + executor.getName()); - } - executors.add(executor); } } @@ -274,8 +213,14 @@ public class ScheduleManager /******************************************************************************* ** *******************************************************************************/ - private void startProcess(QProcessMetaData process, Map backendVariantData) + @Override + public void setupProcess(QProcessMetaData process, Map backendVariantData, boolean allowedToStart) { + if(!allowedToStart) + { + return; + } + Runnable runProcess = () -> { SchedulerUtils.runProcess(qInstance, sessionSupplier, process, backendVariantData); @@ -284,11 +229,6 @@ public class ScheduleManager StandardScheduledExecutor executor = new StandardScheduledExecutor(runProcess); executor.setName("process:" + process.getName()); setScheduleInExecutor(process.getSchedule(), executor); - if(!executor.start()) - { - LOG.warn("executor.start return false for: " + executor.getName()); - } - executors.add(executor); } @@ -363,22 +303,40 @@ public class ScheduleManager /******************************************************************************* ** *******************************************************************************/ - public void stopAsync() + static void resetSingleton() { - for(StandardScheduledExecutor scheduledExecutor : executors) - { - scheduledExecutor.stopAsync(); - } + simpleScheduler = null; } /******************************************************************************* - ** + ** Getter for schedulerName *******************************************************************************/ - static void resetSingleton() + public String getSchedulerName() { - scheduleManager = null; + return (this.schedulerName); + } + + + + /******************************************************************************* + ** Setter for schedulerName + *******************************************************************************/ + public void setSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + } + + + + /******************************************************************************* + ** Fluent setter for schedulerName + *******************************************************************************/ + public SimpleScheduler withSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + return (this); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java similarity index 98% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java index 3910e65f..abaea628 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.scheduler; +package com.kingsrook.qqq.backend.core.scheduler.simple; import java.util.concurrent.Executors; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java index 57e0055d..e08b4ac1 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java @@ -43,7 +43,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; -import com.kingsrook.qqq.backend.core.scheduler.StandardScheduledExecutor; +import com.kingsrook.qqq.backend.core.scheduler.simple.StandardScheduledExecutor; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index b296ff25..251d42f7 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; @@ -69,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMeta import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; @@ -367,6 +369,127 @@ class QInstanceValidatorTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test_validateSchedules() + { + String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE; + Supplier baseScheduleMetaData = () -> new QScheduleMetaData() + .withSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME); + + //////////////////////////////////////////////////// + // do our basic schedule validations on a process // + //////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get()), + "either repeatMillis or repeatSeconds or cronExpression must be set"); + + String validCronString = "* * * * * ?"; + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "both a repeat time and cronExpression may not be set"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatSeconds(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "both a repeat time and cronExpression may not be set"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatSeconds(1) + .withRepeatMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "both a repeat time and cronExpression may not be set"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withInitialDelaySeconds(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "cron schedule may not have an initial delay"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withInitialDelayMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "cron schedule may not have an initial delay"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withInitialDelaySeconds(1) + .withInitialDelayMillis(1) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "cron schedule may not have an initial delay"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression(validCronString)), + "must specify a cronTimeZoneId"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression(validCronString) + .withCronTimeZoneId("foobar")), + "unrecognized cronTimeZoneId: foobar"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression("* * * * * *") + .withCronTimeZoneId("UTC")), + "invalid cron expression: Support for specifying both"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withCronExpression("x") + .withCronTimeZoneId("UTC")), + "invalid cron expression: Illegal cron expression format"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withRepeatSeconds(10) + .withCronTimeZoneId("UTC")), + "non-cron schedule must not specify a cronTimeZoneId"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withSchedulerName(null) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "is missing a scheduler name"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get() + .withSchedulerName("not-a-scheduler") + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "referencing an unknown scheduler name: not-a-scheduler"); + + ///////////////////////////////// + // validate some success cases // + ///////////////////////////////// + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withRepeatSeconds(1))); + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withRepeatMillis(1))); + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("UTC"))); + assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("America/New_York"))); + + ////////////////////////////////////////////////////////////////// + // make sure automation providers get their schedules validated // + ////////////////////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> qInstance.getAutomationProvider(TestUtils.POLLING_AUTOMATION).withSchedule(baseScheduleMetaData.get() + .withSchedulerName(null) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "is missing a scheduler name"); + + ///////////////////////////////////////////////////////////// + // make sure queue providers get their schedules validated // + ///////////////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> ((SQSQueueProviderMetaData)qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER)).withSchedule(baseScheduleMetaData.get() + .withSchedulerName(null) + .withCronExpression(validCronString) + .withCronTimeZoneId("UTC")), + "is missing a scheduler name"); + + } + + + /******************************************************************************* ** Test that a table with no fields fails. ** diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java new file mode 100644 index 00000000..42fc18c5 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java @@ -0,0 +1,63 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import java.util.Properties; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.quartz.SchedulerException; + + +/******************************************************************************* + ** Unit test for QuartzScheduler + *******************************************************************************/ +class QuartzSchedulerTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws SchedulerException + { + Properties quartzProperties = new Properties(); + quartzProperties.put("", ""); + quartzProperties.put("org.quartz.scheduler.instanceName", "TestScheduler"); + quartzProperties.put("org.quartz.threadPool.threadCount", "3"); + quartzProperties.put("org.quartz.jobStore.class", "org.quartz.simpl.RAMJobStore"); + QuartzScheduler.initInstance(QContext.getQInstance(), "TestScheduler", quartzProperties, QContext::getQSession); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java similarity index 85% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java index b164376f..8f1aa280 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC + * Copyright (C) 2021-2024. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.scheduler; +package com.kingsrook.qqq.backend.core.scheduler.simple; import java.util.List; @@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -48,7 +49,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* ** Unit test for ScheduleManager *******************************************************************************/ -class ScheduleManagerTest extends BaseTest +class SimpleSchedulerTest extends BaseTest { /******************************************************************************* @@ -57,7 +58,7 @@ class ScheduleManagerTest extends BaseTest @AfterEach void afterEach() { - ScheduleManager.resetSingleton(); + SimpleScheduler.resetSingleton(); } @@ -69,12 +70,13 @@ class ScheduleManagerTest extends BaseTest void testStartAndStop() { QInstance qInstance = QContext.getQInstance(); - ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance); - scheduleManager.start(); + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); + simpleScheduler.setSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME); + simpleScheduler.start(); - assertThat(scheduleManager.getExecutors()).isNotEmpty(); + assertThat(simpleScheduler.getExecutors()).isNotEmpty(); - scheduleManager.stopAsync(); + simpleScheduler.stopAsync(); } @@ -101,11 +103,12 @@ class ScheduleManagerTest extends BaseTest BasicStep.counter = 0; - ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance); - scheduleManager.setSessionSupplier(QSession::new); - scheduleManager.start(); + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); + simpleScheduler.setSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME); + simpleScheduler.setSessionSupplier(QSession::new); + simpleScheduler.start(); SleepUtils.sleep(50, TimeUnit.MILLISECONDS); - scheduleManager.stopAsync(); + simpleScheduler.stopAsync(); System.out.println("Ran: " + BasicStep.counter + " times"); assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 2e3a937e..fde45605 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -92,6 +92,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView; import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple.SimpleSchedulerMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; @@ -176,6 +179,8 @@ public class TestUtils public static final String SECURITY_KEY_TYPE_STORE_NULL_BEHAVIOR = "storeNullBehavior"; public static final String SECURITY_KEY_TYPE_INTERNAL_OR_EXTERNAL = "internalOrExternal"; + public static final String SIMPLE_SCHEDULER_NAME = "simpleScheduler"; + /******************************************************************************* @@ -237,11 +242,23 @@ public class TestUtils defineWidgets(qInstance); defineApps(qInstance); + qInstance.addScheduler(defineSimpleScheduler()); + return (qInstance); } + /******************************************************************************* + ** + *******************************************************************************/ + private static QSchedulerMetaData defineSimpleScheduler() + { + return new SimpleSchedulerMetaData().withName(SIMPLE_SCHEDULER_NAME); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -349,7 +366,10 @@ public class TestUtils private static QAutomationProviderMetaData definePollingAutomationProvider() { return (new PollingAutomationProviderMetaData() - .withName(POLLING_AUTOMATION)); + .withName(POLLING_AUTOMATION) + .withSchedule(new QScheduleMetaData() + .withSchedulerName(SIMPLE_SCHEDULER_NAME) + .withRepeatSeconds(60))); } @@ -1313,7 +1333,10 @@ public class TestUtils .withAccessKey(accessKey) .withSecretKey(secretKey) .withRegion(region) - .withBaseURL(baseURL)); + .withBaseURL(baseURL) + .withSchedule(new QScheduleMetaData() + .withRepeatSeconds(60) + .withSchedulerName(SIMPLE_SCHEDULER_NAME))); } From c054bf5dddfa0d2cd4ece74fc312c0e1e91d401a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 12:08:41 -0500 Subject: [PATCH 28/72] CE-936 - Fix missing javadoc --- .../kingsrook/qqq/backend/core/context/CapturedContext.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java index b1a80fd5..7cf1aea6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java @@ -34,6 +34,9 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; *******************************************************************************/ public record CapturedContext(QInstance qInstance, QSession qSession, QBackendTransaction qBackendTransaction, Stack actionStack) { + /******************************************************************************* + ** Simpler constructor + *******************************************************************************/ public CapturedContext(QInstance qInstance, QSession qSession) { this(qInstance, qSession, null, null); From 3684101259317d03226cb001a42c5d566d50ff42 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 12:09:24 -0500 Subject: [PATCH 29/72] CE-936 - Fix anti-merge commit from previous --- .../module/rdbms/jdbc/ConnectionManager.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java index ec62895b..27d5adba 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java @@ -40,7 +40,6 @@ public class ConnectionManager *******************************************************************************/ public Connection getConnection(RDBMSBackendMetaData backend) throws SQLException { - String jdbcURL; String jdbcURL = getJdbcUrl(backend); return DriverManager.getConnection(jdbcURL, backend.getUsername(), backend.getPassword()); } @@ -74,23 +73,8 @@ public class ConnectionManager { if(StringUtils.hasContent(backend.getJdbcUrl())) { - jdbcURL = backend.getJdbcUrl(); return backend.getJdbcUrl(); } - else - { - switch(backend.getVendor()) - { - // TODO aws-mysql-jdbc driver not working when running on AWS - // jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=CONVERT_TO_NULL"; - case "aurora" -> jdbcURL = "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false"; - case "mysql" -> jdbcURL = "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull"; - case "h2" -> jdbcURL = "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1"; - default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor()); - } - } - - return DriverManager.getConnection(jdbcURL, backend.getUsername(), backend.getPassword()); return switch(backend.getVendor()) { From 11d33c6d06cdd87da79f025e4128f0d5caa40f73 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 13:39:12 -0500 Subject: [PATCH 30/72] CE-936 - Make test pass (by starting simple scheduler via QScheduleManager) --- .../scheduler/simple/SimpleSchedulerTest.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java index 8f1aa280..fca2dfaa 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java @@ -28,7 +28,6 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; @@ -38,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; @@ -67,9 +67,13 @@ class SimpleSchedulerTest extends BaseTest ** *******************************************************************************/ @Test - void testStartAndStop() + void testStartAndStop() throws QException { - QInstance qInstance = QContext.getQInstance(); + QInstance qInstance = QContext.getQInstance(); + + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> QContext.getQSession()); + qScheduleManager.start(); + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); simpleScheduler.setSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME); simpleScheduler.start(); @@ -85,7 +89,7 @@ class SimpleSchedulerTest extends BaseTest ** *******************************************************************************/ @Test - void testScheduledProcess() throws QInstanceValidationException + void testScheduledProcess() throws QException { QInstance qInstance = QContext.getQInstance(); new QInstanceValidator().validate(qInstance); @@ -95,7 +99,10 @@ class SimpleSchedulerTest extends BaseTest qInstance.addProcess( new QProcessMetaData() .withName("testScheduledProcess") - .withSchedule(new QScheduleMetaData().withRepeatMillis(2).withInitialDelaySeconds(0)) + .withSchedule(new QScheduleMetaData() + .withSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME) + .withRepeatMillis(2) + .withInitialDelaySeconds(0)) .withStepList(List.of(new QBackendStepMetaData() .withName("step") .withCode(new QCodeReference(BasicStep.class)))) @@ -103,6 +110,9 @@ class SimpleSchedulerTest extends BaseTest BasicStep.counter = 0; + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> QContext.getQSession()); + qScheduleManager.start(); + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); simpleScheduler.setSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME); simpleScheduler.setSessionSupplier(QSession::new); From 2c4fc6a0d4e3457508965006fa9f69f24c74c6cd Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 13:49:10 -0500 Subject: [PATCH 31/72] CE-936 - Update start method to actually start schedulers; add stop & stopAsync --- .../core/scheduler/QScheduleManager.java | 26 ++++++++++++++++++- .../scheduler/simple/SimpleSchedulerTest.java | 14 +++++----- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index 34302e23..3978def0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -127,6 +127,31 @@ public class QScheduleManager // ensure that everything which should be scheduled is scheduled, in the appropriate scheduler // ///////////////////////////////////////////////////////////////////////////////////////////////// QContext.withTemporaryContext(new CapturedContext(qInstance, systemUserSessionSupplier.get()), () -> setupSchedules()); + + ////////////////////////// + // start each scheduler // + ////////////////////////// + schedulers.values().forEach(s -> s.start()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void stop() + { + schedulers.values().forEach(s -> s.stop()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void stopAsync() + { + schedulers.values().forEach(s -> s.stopAsync()); } @@ -274,5 +299,4 @@ public class QScheduleManager return (scheduler); } - } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java index fca2dfaa..392cbac8 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java @@ -110,18 +110,18 @@ class SimpleSchedulerTest extends BaseTest BasicStep.counter = 0; - QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> QContext.getQSession()); + QSession qSession = QContext.getQSession(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); qScheduleManager.start(); - SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); - simpleScheduler.setSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME); - simpleScheduler.setSessionSupplier(QSession::new); - simpleScheduler.start(); + ////////////////////////////////////////////////// + // give a moment for the job to run a few times // + ////////////////////////////////////////////////// SleepUtils.sleep(50, TimeUnit.MILLISECONDS); - simpleScheduler.stopAsync(); + qScheduleManager.stopAsync(); System.out.println("Ran: " + BasicStep.counter + " times"); - assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice"); + assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice (but only ran [" + BasicStep.counter + "] time(s)."); } From d09b12ca5bdf3dee914121643b02ea871c2ba658 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 14:05:03 -0500 Subject: [PATCH 32/72] Add unInit, to fix leaked state between tests. --- .../qqq/backend/core/scheduler/QScheduleManager.java | 12 ++++++++++++ .../core/scheduler/simple/SimpleSchedulerTest.java | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index 3978def0..781250eb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -299,4 +299,16 @@ public class QScheduleManager return (scheduler); } + + + + /******************************************************************************* + ** reset the singleton instance (to null); clear the map of schedulers. + ** Not clear it's ever useful to call in main-code - but can be used for tests. + *******************************************************************************/ + public void unInit() + { + qScheduleManager = null; + schedulers.clear(); + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java index 392cbac8..9a8e8ad0 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java @@ -80,7 +80,8 @@ class SimpleSchedulerTest extends BaseTest assertThat(simpleScheduler.getExecutors()).isNotEmpty(); - simpleScheduler.stopAsync(); + qScheduleManager.stop(); + qScheduleManager.unInit(); } @@ -119,6 +120,7 @@ class SimpleSchedulerTest extends BaseTest ////////////////////////////////////////////////// SleepUtils.sleep(50, TimeUnit.MILLISECONDS); qScheduleManager.stopAsync(); + qScheduleManager.unInit(); System.out.println("Ran: " + BasicStep.counter + " times"); assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice (but only ran [" + BasicStep.counter + "] time(s)."); From 7181643abb918e9de00ff5971ad6801b010aec41 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 15:46:15 -0500 Subject: [PATCH 33/72] CE-936 - Move log4j-slf4j-impl from mongo to core --- qqq-backend-core/pom.xml | 7 +++++++ qqq-backend-module-mongodb/pom.xml | 5 ----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index d7c1e4cf..38bad7f5 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -169,6 +169,13 @@ 2.3.2 + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.23.0 + + org.apache.maven.plugins diff --git a/qqq-backend-module-mongodb/pom.xml b/qqq-backend-module-mongodb/pom.xml index 58ddb23b..4aa23c36 100644 --- a/qqq-backend-module-mongodb/pom.xml +++ b/qqq-backend-module-mongodb/pom.xml @@ -50,11 +50,6 @@ mongodb-driver-sync 4.11.1 - - org.apache.logging.log4j - log4j-slf4j-impl - 2.23.0 - org.testcontainers From b093ff5ecef5bcfa27108796bae35ef4cb9fb475 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 15:47:15 -0500 Subject: [PATCH 34/72] CE-936 - Test coverage on quartz code --- .../core/scheduler/QScheduleManager.java | 1 + .../core/scheduler/QSchedulerInterface.java | 29 +- .../quartz/QuartzJobAndTriggerWrapper.java | 34 +++ .../scheduler/quartz/QuartzScheduler.java | 47 ++- .../scheduler/quartz/QuartzSqsPollerJob.java | 17 +- .../quartz/QuartzTableAutomationsJob.java | 18 +- .../processes/PauseQuartzJobsProcess.java | 2 +- .../processes/ResumeQuartzJobsProcess.java | 2 +- .../scheduler/simple/SimpleScheduler.java | 14 +- .../scheduler/quartz/QuartzSchedulerTest.java | 112 ++++++- .../scheduler/quartz/QuartzTestUtils.java | 96 ++++++ .../processes/QuartzJobsProcessTest.java | 277 ++++++++++++++++++ .../scheduler/simple/SimpleSchedulerTest.java | 7 +- 13 files changed, 614 insertions(+), 42 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobAndTriggerWrapper.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index 781250eb..b88e7ceb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -309,6 +309,7 @@ public class QScheduleManager public void unInit() { qScheduleManager = null; + schedulers.values().forEach(s -> s.unInit()); schedulers.clear(); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java index 62404fb0..87d9cea2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java @@ -34,12 +34,26 @@ import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMeta *******************************************************************************/ public interface QSchedulerInterface { + /******************************************************************************* + ** + *******************************************************************************/ + void setupProcess(QProcessMetaData process, Map backendVariantData, boolean allowedToStart); /******************************************************************************* ** *******************************************************************************/ void setupSqsProvider(SQSQueueProviderMetaData queueProvider, boolean allowedToStart); + /******************************************************************************* + ** + *******************************************************************************/ + void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStart); + + /******************************************************************************* + ** + *******************************************************************************/ + void start(); + /******************************************************************************* ** *******************************************************************************/ @@ -51,19 +65,12 @@ public interface QSchedulerInterface void stop(); /******************************************************************************* - ** + ** Handle a whole shutdown of the scheduler system (e.g., between unit tests). *******************************************************************************/ - void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStart); + default void unInit() + { - /******************************************************************************* - ** - *******************************************************************************/ - void setupProcess(QProcessMetaData process, Map backendVariantData, boolean allowedToStart); - - /******************************************************************************* - ** - *******************************************************************************/ - void start(); + } /******************************************************************************* ** let the scheduler know when the schedule manager is at the start of setting up schedules. diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobAndTriggerWrapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobAndTriggerWrapper.java new file mode 100644 index 00000000..62ec6cb7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobAndTriggerWrapper.java @@ -0,0 +1,34 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import org.quartz.JobDetail; +import org.quartz.Trigger; + + +/******************************************************************************* + ** + *******************************************************************************/ +public record QuartzJobAndTriggerWrapper(JobDetail jobDetail, Trigger trigger, Trigger.TriggerState triggerState) +{ +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java index da9fe9ca..dee8cf9a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.scheduler.quartz; import java.io.Serializable; import java.time.Duration; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -250,7 +251,7 @@ public class QuartzScheduler implements QSchedulerInterface } else { - long intervalMillis = Objects.requireNonNullElse(scheduleMetaData.getRepeatMillis(), scheduleMetaData.getRepeatSeconds() * 1000); + long intervalMillis = Objects.requireNonNullElseGet(scheduleMetaData.getRepeatMillis(), () -> scheduleMetaData.getRepeatSeconds() * 1000); scheduleBuilder = SimpleScheduleBuilder.simpleSchedule() .withIntervalInMilliseconds(intervalMillis) .repeatForever(); @@ -376,6 +377,8 @@ public class QuartzScheduler implements QSchedulerInterface this.scheduler.scheduleJob(jobDetail, trigger); LOG.info("Scheduled new job: " + jobKey); } + + // todo - think about... clear memoization - but - when this is used in bulk, that's when we want the memo! } @@ -499,4 +502,46 @@ public class QuartzScheduler implements QSchedulerInterface { this.scheduler.resumeJob(new JobKey(jobName, groupName)); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + List queryQuartz() throws SchedulerException + { + List rs = new ArrayList<>(); + List jobGroupNames = scheduler.getJobGroupNames(); + + for(String group : jobGroupNames) + { + Set jobKeys = scheduler.getJobKeys(GroupMatcher.groupEquals(group)); + for(JobKey jobKey : jobKeys) + { + JobDetail jobDetail = scheduler.getJobDetail(jobKey); + List triggersOfJob = scheduler.getTriggersOfJob(jobKey); + for(Trigger trigger : triggersOfJob) + { + Trigger.TriggerState triggerState = scheduler.getTriggerState(trigger.getKey()); + rs.add(new QuartzJobAndTriggerWrapper(jobDetail, trigger, triggerState)); + } + } + } + + return (rs); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unInit() + { + /////////////////////////////////////////////////// + // resetting the singleton should be sufficient! // + /////////////////////////////////////////////////// + quartzScheduler = null; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java index 7270d7fa..140a90d4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java @@ -51,12 +51,15 @@ public class QuartzSqsPollerJob implements Job @Override public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { + String queueProviderName = null; + String queueName = null; + try { - JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); - String queueProviderName = jobDataMap.getString("queueProviderName"); - String queueName = jobDataMap.getString("queueName"); - QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); + queueProviderName = jobDataMap.getString("queueProviderName"); + queueName = jobDataMap.getString("queueName"); + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); sqsQueuePoller.setQueueProviderMetaData((SQSQueueProviderMetaData) qInstance.getQueueProvider(queueProviderName)); @@ -67,9 +70,13 @@ public class QuartzSqsPollerJob implements Job ///////////// // run it. // ///////////// - LOG.debug("Running quartz SQS Poller", logPair("queueName", queueName)); + LOG.debug("Running quartz SQS Poller", logPair("queueName", queueName), logPair("queueProviderName", queueProviderName)); sqsQueuePoller.run(); } + catch(Exception e) + { + LOG.warn("Error running SQS Poller", e, logPair("queueName", queueName), logPair("queueProviderName", queueProviderName)); + } finally { QContext.clear(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java index 4f4f244b..9d34bc22 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java @@ -51,13 +51,17 @@ public class QuartzTableAutomationsJob implements Job @Override public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { + String tableName = null; + String automationProviderName = null; + AutomationStatus automationStatus = null; + try { - JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); - String tableName = jobDataMap.getString("tableName"); - String automationProviderName = jobDataMap.getString("automationProviderName"); - AutomationStatus automationStatus = AutomationStatus.valueOf(jobDataMap.getString("automationStatus")); - QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); + tableName = jobDataMap.getString("tableName"); + automationProviderName = jobDataMap.getString("automationProviderName"); + automationStatus = AutomationStatus.valueOf(jobDataMap.getString("automationStatus")); + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); PollingAutomationPerTableRunner.TableActionsInterface tableAction = new PollingAutomationPerTableRunner.TableActions(tableName, automationStatus); PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProviderName, QuartzScheduler.getInstance().getSessionSupplier(), tableAction); @@ -68,6 +72,10 @@ public class QuartzTableAutomationsJob implements Job LOG.debug("Running Table Automations", logPair("tableName", tableName), logPair("automationStatus", automationStatus)); runner.run(); } + catch(Exception e) + { + LOG.warn("Error running Table Automations", e, logPair("tableName", tableName), logPair("automationStatus", automationStatus)); + } finally { QContext.clear(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java index fbaa4679..96f033cd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java @@ -77,7 +77,7 @@ public class PauseQuartzJobsProcess extends AbstractLoadStep implements MetaData QuartzScheduler instance = QuartzScheduler.getInstance(); for(QRecord record : runBackendStepInput.getRecords()) { - instance.pauseJob(record.getValueString("JOB_NAME"), record.getValueString("GROUP_NAME")); + instance.pauseJob(record.getValueString("jobName"), record.getValueString("groupName")); } } catch(Exception e) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java index a4ceff24..e7215452 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java @@ -77,7 +77,7 @@ public class ResumeQuartzJobsProcess extends AbstractLoadStep implements MetaDat QuartzScheduler instance = QuartzScheduler.getInstance(); for(QRecord record : runBackendStepInput.getRecords()) { - instance.resumeJob(record.getValueString("JOB_NAME"), record.getValueString("GROUP_NAME")); + instance.resumeJob(record.getValueString("jobName"), record.getValueString("groupName")); } } catch(Exception e) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java index ed215833..e87e039a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java @@ -305,7 +305,6 @@ public class SimpleScheduler implements QSchedulerInterface *******************************************************************************/ static void resetSingleton() { - simpleScheduler = null; } @@ -339,4 +338,17 @@ public class SimpleScheduler implements QSchedulerInterface return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unInit() + { + ////////////////////////////////////////////////// + // resetting the singleton should be sufficient // + ////////////////////////////////////////////////// + simpleScheduler = null; + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java index 42fc18c5..a69e2e14 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java @@ -22,12 +22,29 @@ package com.kingsrook.qqq.backend.core.scheduler.quartz; -import java.util.Properties; +import java.util.List; +import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.context.QContext; -import org.junit.jupiter.api.BeforeEach; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.logging.log4j.Level; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -import org.quartz.SchedulerException; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* @@ -35,18 +52,14 @@ import org.quartz.SchedulerException; *******************************************************************************/ class QuartzSchedulerTest extends BaseTest { + /******************************************************************************* ** *******************************************************************************/ - @BeforeEach - void beforeEach() throws SchedulerException + @AfterEach + void afterEach() { - Properties quartzProperties = new Properties(); - quartzProperties.put("", ""); - quartzProperties.put("org.quartz.scheduler.instanceName", "TestScheduler"); - quartzProperties.put("org.quartz.threadPool.threadCount", "3"); - quartzProperties.put("org.quartz.jobStore.class", "org.quartz.simpl.RAMJobStore"); - QuartzScheduler.initInstance(QContext.getQInstance(), "TestScheduler", quartzProperties, QContext::getQSession); + QScheduleManager.getInstance().unInit(); } @@ -55,9 +68,84 @@ class QuartzSchedulerTest extends BaseTest ** *******************************************************************************/ @Test - void test() + void test() throws Exception { + try + { + QInstance qInstance = QContext.getQInstance(); + QuartzTestUtils.setupInstanceForQuartzTests(); + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // set these runners to use collecting logger, so we can assert that they did run, and didn't throw // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + QCollectingLogger quartzSqsPollerJobLog = QLogger.activateCollectingLoggerForClass(QuartzSqsPollerJob.class); + QCollectingLogger quartzTableAutomationsJobLog = QLogger.activateCollectingLoggerForClass(QuartzTableAutomationsJob.class); + + ////////////////////////////////////////// + // add a process we can run and observe // + ////////////////////////////////////////// + qInstance.addProcess(new QProcessMetaData() + .withName("testScheduledProcess") + .withSchedule(new QScheduleMetaData() + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withRepeatMillis(2) + .withInitialDelaySeconds(0)) + .withStepList(List.of(new QBackendStepMetaData() + .withName("step") + .withCode(new QCodeReference(BasicStep.class))))); + + ////////////////////////////////////////////////////////////////////////////// + // start the schedule manager, which will schedule things, and start quartz // + ////////////////////////////////////////////////////////////////////////////// + QSession qSession = QContext.getQSession(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); + qScheduleManager.start(); + + ////////////////////////////////////////////////// + // give a moment for the job to run a few times // + ////////////////////////////////////////////////// + SleepUtils.sleep(50, TimeUnit.MILLISECONDS); + qScheduleManager.stopAsync(); + + System.out.println("Ran: " + BasicStep.counter + " times"); + assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice (but only ran [" + BasicStep.counter + "] time(s)."); + + ////////////////////////////////////////////////////// + // make sure poller ran, and didn't issue any warns // + ////////////////////////////////////////////////////// + assertThat(quartzSqsPollerJobLog.getCollectedMessages()) + .anyMatch(m -> m.getLevel().equals(Level.DEBUG) && m.getMessage().contains("Running quartz SQS Poller")) + .noneMatch(m -> m.getLevel().equals(Level.WARN)); + + ////////////////////////////////////////////////////// + // make sure poller ran, and didn't issue any warns // + ////////////////////////////////////////////////////// + assertThat(quartzTableAutomationsJobLog.getCollectedMessages()) + .anyMatch(m -> m.getLevel().equals(Level.DEBUG) && m.getMessage().contains("Running Table Automations")) + .noneMatch(m -> m.getLevel().equals(Level.WARN)); + } + finally + { + QLogger.deactivateCollectingLoggerForClass(QuartzSqsPollerJob.class); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class BasicStep implements BackendStep + { + static int counter = 0; + + + + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + counter++; + } } } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java new file mode 100644 index 00000000..702207f3 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java @@ -0,0 +1,96 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz; + + +import java.util.List; +import java.util.Properties; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData; +import org.quartz.SchedulerException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuartzTestUtils +{ + public final static String QUARTZ_SCHEDULER_NAME = "TestQuartzScheduler"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Properties getQuartzProperties() + { + Properties quartzProperties = new Properties(); + quartzProperties.put("org.quartz.scheduler.instanceName", QUARTZ_SCHEDULER_NAME); + quartzProperties.put("org.quartz.threadPool.threadCount", "3"); + quartzProperties.put("org.quartz.jobStore.class", "org.quartz.simpl.RAMJobStore"); + return (quartzProperties); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void setupInstanceForQuartzTests() + { + QInstance qInstance = QContext.getQInstance(); + + /////////////////////////////////////////////////// + // remove the simple scheduler from the instance // + /////////////////////////////////////////////////// + qInstance.getSchedulers().clear(); + + //////////////////////////////////////////////////////// + // add the quartz scheduler meta-data to the instance // + //////////////////////////////////////////////////////// + qInstance.addScheduler(new QuartzSchedulerMetaData() + .withProperties(getQuartzProperties()) + .withName(QUARTZ_SCHEDULER_NAME)); + + //////////////////////////////////////////////////////////////////////////////// + // set the queue providers & automation providers to use the quartz scheduler // + //////////////////////////////////////////////////////////////////////////////// + qInstance.getAutomationProviders().values() + .forEach(ap -> ap.getSchedule().setSchedulerName(QUARTZ_SCHEDULER_NAME)); + + qInstance.getQueueProviders().values() + .forEach(qp -> ((SQSQueueProviderMetaData) qp).getSchedule().setSchedulerName(QUARTZ_SCHEDULER_NAME)); + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static List queryQuartz() throws SchedulerException + { + return QuartzScheduler.getInstance().queryQuartz(); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java new file mode 100644 index 00000000..760d12c3 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java @@ -0,0 +1,277 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.quartz.SchedulerException; +import org.quartz.Trigger; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit tests for the various quartz management processes + *******************************************************************************/ +class QuartzJobsProcessTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws QException + { + QInstance qInstance = QContext.getQInstance(); + qInstance.addTable(new QTableMetaData() + .withName("quartzTriggers") + .withBackendName(TestUtils.MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.LONG))); + MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, QuartzScheduler.class.getPackageName()); + + QuartzTestUtils.setupInstanceForQuartzTests(); + + ////////////////////////////////////////////////////////////////////////////// + // start the schedule manager, which will schedule things, and start quartz // + ////////////////////////////////////////////////////////////////////////////// + QSession qSession = QContext.getQSession(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); + qScheduleManager.start(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QScheduleManager.getInstance().stop(); + QScheduleManager.getInstance().unInit(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPauseAllQuartzJobs() throws QException, SchedulerException + { + //////////////////////////////////////// + // make sure nothing starts as paused // + //////////////////////////////////////// + assertNoneArePaused(); + + /////////////////////////////// + // run the pause-all process // + /////////////////////////////// + RunProcessInput input = new RunProcessInput(); + input.setProcessName(PauseAllQuartzJobsProcess.class.getSimpleName()); + new RunProcessAction().execute(input); + + ////////////////////////////////////// + // assert everything becomes paused // + ////////////////////////////////////// + assertAllArePaused(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testResumeAllQuartzJobs() throws QException, SchedulerException + { + /////////////////////////////// + // run the pause-all process // + /////////////////////////////// + RunProcessInput input = new RunProcessInput(); + input.setProcessName(PauseAllQuartzJobsProcess.class.getSimpleName()); + new RunProcessAction().execute(input); + + ////////////////////////////////////// + // assert everything becomes paused // + ////////////////////////////////////// + assertAllArePaused(); + + //////////////////// + // run resume all // + //////////////////// + input = new RunProcessInput(); + input.setProcessName(ResumeAllQuartzJobsProcess.class.getSimpleName()); + new RunProcessAction().execute(input); + + //////////////////////////////////////// + // make sure nothing ends up as paused // + //////////////////////////////////////// + assertNoneArePaused(); + + //////////////////// + // pause just one // + //////////////////// + List quartzJobAndTriggerWrappers = QuartzTestUtils.queryQuartz(); + new InsertAction().execute(new InsertInput("quartzTriggers").withRecord(new QRecord() + .withValue("jobName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()) + .withValue("groupName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getGroup()) + )); + + input = new RunProcessInput(); + input.setProcessName(PauseQuartzJobsProcess.class.getSimpleName()); + input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter())); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + ///////////////////////////////////////////////////////// + // make sure at least 1 is paused, some are not paused // + ///////////////////////////////////////////////////////// + assertAnyAre(Trigger.TriggerState.PAUSED); + assertAnyAreNot(Trigger.TriggerState.PAUSED); + + ////////////////////////// + // run resume all again // + ////////////////////////// + input = new RunProcessInput(); + input.setProcessName(ResumeAllQuartzJobsProcess.class.getSimpleName()); + new RunProcessAction().execute(input); + + //////////////////////////////////////// + // make sure nothing ends up as paused // + //////////////////////////////////////// + assertNoneArePaused(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPauseOneResumeOne() throws QException, SchedulerException + { + ///////////////////////////////////// + // make sure nothing starts paused // + ///////////////////////////////////// + assertNoneArePaused(); + + //////////////////// + // pause just one // + //////////////////// + List quartzJobAndTriggerWrappers = QuartzTestUtils.queryQuartz(); + new InsertAction().execute(new InsertInput("quartzTriggers").withRecord(new QRecord() + .withValue("jobName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()) + .withValue("groupName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getGroup()) + )); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName(PauseQuartzJobsProcess.class.getSimpleName()); + input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter())); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + ///////////////////////////////////////////////////////// + // make sure at least 1 is paused, some are not paused // + ///////////////////////////////////////////////////////// + assertAnyAre(Trigger.TriggerState.PAUSED); + assertAnyAreNot(Trigger.TriggerState.PAUSED); + + ///////////////////////////////////////////////////////////////////////////// + // now resume the same one (will still be only row in our in-memory table) // + ///////////////////////////////////////////////////////////////////////////// + input = new RunProcessInput(); + input.setProcessName(ResumeQuartzJobsProcess.class.getSimpleName()); + input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter())); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + ////////////////////////////////////// + // make sure nothing ends up paused // + ////////////////////////////////////// + assertNoneArePaused(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void assertAnyAre(Trigger.TriggerState triggerState) throws SchedulerException + { + assertThat(QuartzTestUtils.queryQuartz()).anyMatch(qjtw -> qjtw.triggerState().equals(triggerState)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void assertAnyAreNot(Trigger.TriggerState triggerState) throws SchedulerException + { + assertThat(QuartzTestUtils.queryQuartz()).anyMatch(qjtw -> !qjtw.triggerState().equals(triggerState)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void assertNoneArePaused() throws SchedulerException + { + assertThat(QuartzTestUtils.queryQuartz()).noneMatch(qjtw -> qjtw.triggerState().equals(Trigger.TriggerState.PAUSED)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void assertAllArePaused() throws SchedulerException + { + assertThat(QuartzTestUtils.queryQuartz()).allMatch(qjtw -> qjtw.triggerState().equals(Trigger.TriggerState.PAUSED)); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java index 9a8e8ad0..f0f1af03 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java @@ -58,7 +58,7 @@ class SimpleSchedulerTest extends BaseTest @AfterEach void afterEach() { - SimpleScheduler.resetSingleton(); + QScheduleManager.getInstance().unInit(); } @@ -81,7 +81,6 @@ class SimpleSchedulerTest extends BaseTest assertThat(simpleScheduler.getExecutors()).isNotEmpty(); qScheduleManager.stop(); - qScheduleManager.unInit(); } @@ -106,8 +105,7 @@ class SimpleSchedulerTest extends BaseTest .withInitialDelaySeconds(0)) .withStepList(List.of(new QBackendStepMetaData() .withName("step") - .withCode(new QCodeReference(BasicStep.class)))) - ); + .withCode(new QCodeReference(BasicStep.class))))); BasicStep.counter = 0; @@ -120,7 +118,6 @@ class SimpleSchedulerTest extends BaseTest ////////////////////////////////////////////////// SleepUtils.sleep(50, TimeUnit.MILLISECONDS); qScheduleManager.stopAsync(); - qScheduleManager.unInit(); System.out.println("Ran: " + BasicStep.counter + " times"); assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice (but only ran [" + BasicStep.counter + "] time(s)."); From 03b93658d5bebfe1e11d63754c3c32dc97130435 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Mar 2024 18:26:47 -0500 Subject: [PATCH 35/72] CE-936 - Add test on RDBMSTableMetaDataBuilder (and support in it for h2, and bug-fix to use fieldName, not columnName, for primaryKey) --- .../metadata/RDBMSTableMetaDataBuilder.java | 18 +++-- .../RDBMSTableMetaDataBuilderTest.java | 70 +++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilderTest.java diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java index 3d1e774f..7fc556d4 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilder.java @@ -61,6 +61,7 @@ public class RDBMSTableMetaDataBuilder typeMap.put("VARBINARY", QFieldType.BLOB); typeMap.put("MEDIUMBLOB", QFieldType.BLOB); typeMap.put("NUMERIC", QFieldType.INTEGER); + typeMap.put("INTEGER", QFieldType.INTEGER); typeMap.put("BIGINT UNSIGNED", QFieldType.LONG); typeMap.put("MEDIUMINT UNSIGNED", QFieldType.INTEGER); typeMap.put("SMALLINT UNSIGNED", QFieldType.INTEGER); @@ -95,9 +96,18 @@ public class RDBMSTableMetaDataBuilder dataTypeMap.put(id, name); } + // todo - for h2, uppercase both db & table names... String databaseName = backendMetaData.getDatabaseName(); // these work for mysql - unclear about other vendors. String schemaName = null; - try(ResultSet tableResultSet = databaseMetaData.getTables(databaseName, schemaName, tableName, null)) + String tableNameForMetaDataQueries = tableName; + + if(backendMetaData.getVendor().equals("h2")) + { + databaseName = databaseName.toUpperCase(); + tableNameForMetaDataQueries = tableName.toUpperCase(); + } + + try(ResultSet tableResultSet = databaseMetaData.getTables(databaseName, schemaName, tableNameForMetaDataQueries, null)) { if(!tableResultSet.next()) { @@ -105,7 +115,7 @@ public class RDBMSTableMetaDataBuilder } } - try(ResultSet columnsResultSet = databaseMetaData.getColumns(databaseName, schemaName, tableName, null)) + try(ResultSet columnsResultSet = databaseMetaData.getColumns(databaseName, schemaName, tableNameForMetaDataQueries, null)) { while(columnsResultSet.next()) { @@ -119,7 +129,7 @@ public class RDBMSTableMetaDataBuilder QFieldType type = typeMap.get(dataTypeName); if(type == null) { - LOG.info("Table " + tableName + " column " + columnName + " has an unampped type: " + dataTypeId + ". Field will not be added to QTableMetaData"); + LOG.info("Table " + tableName + " column " + columnName + " has an unmapped type: " + dataTypeId + ". Field will not be added to QTableMetaData"); continue; } @@ -133,7 +143,7 @@ public class RDBMSTableMetaDataBuilder if("YES".equals(isAutoIncrement)) { - primaryKey = columnName; + primaryKey = qqqFieldName; } } } diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilderTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilderTest.java new file mode 100644 index 00000000..c6b8fc17 --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSTableMetaDataBuilderTest.java @@ -0,0 +1,70 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.model.metadata; + + +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.rdbms.BaseTest; +import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for RDBMSTableMetaDataBuilder + *******************************************************************************/ +class RDBMSTableMetaDataBuilderTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + TestUtils.primeTestDatabase("prime-test-database.sql"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + QBackendMetaData backend = QContext.getQInstance().getBackend(TestUtils.DEFAULT_BACKEND_NAME); + QTableMetaData table = new RDBMSTableMetaDataBuilder().buildTableMetaData((RDBMSBackendMetaData) backend, "order"); + + assertNotNull(table); + assertEquals("order", table.getName()); + assertEquals("id", table.getPrimaryKeyField()); + assertEquals(QFieldType.INTEGER, table.getField(table.getPrimaryKeyField()).getType()); + assertNotNull(table.getField("storeId")); + } + +} \ No newline at end of file From 246984892af338d32d459535eeebc1bf6305ead8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 13 Mar 2024 12:05:13 -0500 Subject: [PATCH 36/72] Don't include not-protected widgets in available-permissions list --- .../backend/core/actions/permissions/PermissionsHelper.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java index cdc2f439..57a368f7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.permissions.DenyBehavior; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; @@ -387,7 +388,10 @@ public class PermissionsHelper { QPermissionRules rules = getEffectivePermissionRules(widgetMetaData, instance); String baseName = getEffectivePermissionBaseName(rules, widgetMetaData.getName()); - addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget"); + if(!rules.getLevel().equals(PermissionLevel.NOT_PROTECTED)) + { + addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget"); + } } return (rs); From 3265d6d842b82982e081c929bf234aaeefb90e04 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 13 Mar 2024 12:08:13 -0500 Subject: [PATCH 37/72] CE-936 - scheduling updates: - move queues & automations to be scheduled (only) at the lower-level (per-queue, per-table) - not at the higher "provider" levels. - update quartz to delete jobs which are no-longer active, at end of QScheduleManager's setup - --- .../PollingAutomationPerTableRunner.java | 23 +-- .../core/instances/QInstanceValidator.java | 23 +-- .../QAutomationProviderMetaData.java | 37 ----- .../queues/SQSQueueProviderMetaData.java | 39 ----- .../scheduleing/QScheduleMetaData.java | 5 +- .../automation/QTableAutomationDetails.java | 34 ++++ .../core/scheduler/QScheduleManager.java | 39 ++++- .../core/scheduler/QSchedulerInterface.java | 9 +- .../scheduler/quartz/QuartzScheduler.java | 152 +++++++++++++----- .../quartz/QuartzTableAutomationsJob.java | 21 ++- .../scheduler/simple/SimpleScheduler.java | 64 +++----- .../simple/StandardScheduledExecutor.java | 6 +- .../PollingAutomationPerTableRunnerTest.java | 4 +- .../instances/QInstanceValidatorTest.java | 16 +- .../scheduler/quartz/QuartzSchedulerTest.java | 86 ++++++++-- .../scheduler/quartz/QuartzTestUtils.java | 17 +- .../processes/QuartzJobsProcessTest.java | 24 ++- .../qqq/backend/core/utils/TestUtils.java | 21 +-- 18 files changed, 385 insertions(+), 235 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java index 2113facc..9b013a85 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java @@ -132,6 +132,11 @@ public class PollingAutomationPerTableRunner implements Runnable *******************************************************************************/ String tableName(); + /******************************************************************************* + ** + *******************************************************************************/ + QTableAutomationDetails tableAutomationDetails(); + /******************************************************************************* ** *******************************************************************************/ @@ -143,7 +148,7 @@ public class PollingAutomationPerTableRunner implements Runnable /******************************************************************************* ** Wrapper for a pair of (tableName, automationStatus) *******************************************************************************/ - public record TableActions(String tableName, AutomationStatus status) implements TableActionsInterface + public record TableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status) implements TableActionsInterface { /******************************************************************************* ** @@ -159,7 +164,7 @@ public class PollingAutomationPerTableRunner implements Runnable ** extended version of TableAction, for sharding use-case - adds the shard ** details. *******************************************************************************/ - public record ShardedTableActions(String tableName, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface + public record ShardedTableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface { /******************************************************************************* ** @@ -198,8 +203,8 @@ public class PollingAutomationPerTableRunner implements Runnable { Serializable shardId = record.getValue(automationDetails.getShardIdFieldName()); String label = record.getValueString(automationDetails.getShardLabelFieldName()); - tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label)); - tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label)); + tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label)); + tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label)); } } catch(Exception e) @@ -209,11 +214,11 @@ public class PollingAutomationPerTableRunner implements Runnable } else { - /////////////////////////////////////////////////////////////////// - // for non-sharded, we just need tabler name & automation status // - /////////////////////////////////////////////////////////////////// - tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS)); - tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + ////////////////////////////////////////////////////////////////// + // for non-sharded, we just need table name & automation status // + ////////////////////////////////////////////////////////////////// + tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 8da52458..bc4dc01d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -417,11 +417,6 @@ public class QInstanceValidator assertCondition(StringUtils.hasContent(sqsQueueProvider.getSecretKey()), "Missing secretKey for SQSQueueProvider: " + name); assertCondition(StringUtils.hasContent(sqsQueueProvider.getBaseURL()), "Missing baseURL for SQSQueueProvider: " + name); assertCondition(StringUtils.hasContent(sqsQueueProvider.getRegion()), "Missing region for SQSQueueProvider: " + name); - - if(assertCondition(sqsQueueProvider.getSchedule() != null, "Missing schedule for SQSQueueProvider: " + name)) - { - validateScheduleMetaData(sqsQueueProvider.getSchedule(), qInstance, "SQSQueueProvider " + name + ", schedule: "); - } } runPlugins(QQueueProviderMetaData.class, queueProvider, qInstance); @@ -440,6 +435,14 @@ public class QInstanceValidator assertCondition(qInstance.getProcesses() != null && qInstance.getProcess(queue.getProcessName()) != null, "Unrecognized processName for queue: " + name); } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // todo - if we have, in the future, a provider that doesn't require schedules per-queue, then make this check conditional // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(assertCondition(queue.getSchedule() != null, "Missing schedule for SQSQueueProvider: " + name)) + { + validateScheduleMetaData(queue.getSchedule(), qInstance, "SQSQueueProvider " + name + ", schedule: "); + } + runPlugins(QQueueMetaData.class, queue, qInstance); }); } @@ -479,11 +482,6 @@ public class QInstanceValidator assertCondition(Objects.equals(name, automationProvider.getName()), "Inconsistent naming for automationProvider: " + name + "/" + automationProvider.getName() + "."); assertCondition(automationProvider.getType() != null, "Missing type for automationProvider: " + name); - if(assertCondition(automationProvider.getSchedule() != null, "Missing schedule for automationProvider: " + name)) - { - validateScheduleMetaData(automationProvider.getSchedule(), qInstance, "automationProvider " + name + ", schedule: "); - } - runPlugins(QAutomationProviderMetaData.class, automationProvider, qInstance); }); } @@ -1026,6 +1024,11 @@ public class QInstanceValidator assertCondition(qInstance.getAutomationProvider(providerName) != null, " has an unrecognized providerName: " + providerName); } + if(assertCondition(automationDetails.getSchedule() != null, prefix + "Missing schedule for automations")) + { + validateScheduleMetaData(automationDetails.getSchedule(), qInstance, prefix + " automationDetails, schedule: "); + } + ////////////////////////////////// // validate the status tracking // ////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java index 6dca9cde..31af5cff 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java @@ -24,7 +24,6 @@ package com.kingsrook.qqq.backend.core.model.metadata.automation; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; /******************************************************************************* @@ -35,8 +34,6 @@ public class QAutomationProviderMetaData implements TopLevelMetaDataInterface private String name; private QAutomationProviderType type; - private QScheduleMetaData schedule; - /******************************************************************************* @@ -107,40 +104,6 @@ public class QAutomationProviderMetaData implements TopLevelMetaDataInterface - /******************************************************************************* - ** Getter for schedule - ** - *******************************************************************************/ - public QScheduleMetaData getSchedule() - { - return schedule; - } - - - - /******************************************************************************* - ** Setter for schedule - ** - *******************************************************************************/ - public void setSchedule(QScheduleMetaData schedule) - { - this.schedule = schedule; - } - - - - /******************************************************************************* - ** Fluent setter for schedule - ** - *******************************************************************************/ - public QAutomationProviderMetaData withSchedule(QScheduleMetaData schedule) - { - this.schedule = schedule; - return (this); - } - - - /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java index 3825b9ff..6184db3f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java @@ -22,9 +22,6 @@ package com.kingsrook.qqq.backend.core.model.metadata.queues; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; - - /******************************************************************************* ** Meta-data for an source of Amazon SQS queues (e.g, an aws account/credential ** set, with a common base URL). @@ -39,8 +36,6 @@ public class SQSQueueProviderMetaData extends QQueueProviderMetaData private String region; private String baseURL; - private QScheduleMetaData schedule; - /******************************************************************************* @@ -201,38 +196,4 @@ public class SQSQueueProviderMetaData extends QQueueProviderMetaData return (this); } - - - /******************************************************************************* - ** Getter for schedule - ** - *******************************************************************************/ - public QScheduleMetaData getSchedule() - { - return schedule; - } - - - - /******************************************************************************* - ** Setter for schedule - ** - *******************************************************************************/ - public void setSchedule(QScheduleMetaData schedule) - { - this.schedule = schedule; - } - - - - /******************************************************************************* - ** Fluent setter for schedule - ** - *******************************************************************************/ - public SQSQueueProviderMetaData withSchedule(QScheduleMetaData schedule) - { - this.schedule = schedule; - return (this); - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java index c7373f1a..8dd19d96 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java @@ -28,11 +28,12 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* ** Meta-data to define scheduled actions within QQQ. ** - ** Initially, only supports repeating jobs, either on a given # of seconds or millis. + ** Supports repeating jobs, either on a given # of seconds or millis, or cron + ** expressions (though cron may not be supported by all schedulers!) + ** ** Can also specify an initialDelay - e.g., to avoid all jobs starting up at the ** same moment. ** - ** In the future we most likely would want to allow cron strings to be added here. *******************************************************************************/ public class QScheduleMetaData { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java index 303e2932..13a57662 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables.automation; import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; /******************************************************************************* @@ -37,6 +38,8 @@ public class QTableAutomationDetails private Integer overrideBatchSize; + private QScheduleMetaData schedule; + private String shardByFieldName; // field in "this" table, to use for sharding private String shardSourceTableName; // name of the table where the shards are defined as rows private String shardLabelFieldName; // field in shard-source-table to use for labeling shards @@ -317,4 +320,35 @@ public class QTableAutomationDetails return (this); } + + /******************************************************************************* + ** Getter for schedule + *******************************************************************************/ + public QScheduleMetaData getSchedule() + { + return (this.schedule); + } + + + + /******************************************************************************* + ** Setter for schedule + *******************************************************************************/ + public void setSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + } + + + + /******************************************************************************* + ** Fluent setter for schedule + *******************************************************************************/ + public QTableAutomationDetails withSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index b88e7ceb..e3edee1e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -24,8 +24,10 @@ package com.kingsrook.qqq.backend.core.scheduler; import java.io.Serializable; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; import com.kingsrook.qqq.backend.core.context.CapturedContext; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -37,12 +39,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; import org.apache.commons.lang.NotImplementedException; @@ -227,6 +231,8 @@ public class QScheduleManager // todo - read dynamic schedules and schedule those things // // e.g., user-scheduled processes, reports // ///////////////////////////////////////////////////////////// + // ScheduledJob scheduledJob = new ScheduledJob(); + // setupScheduledJob(scheduledJob); ////////////////////////////////////////////////////////// // let the schedulers know we're done with this process // @@ -242,7 +248,7 @@ public class QScheduleManager private void setupProcess(QProcessMetaData process, Map backendVariantData) { QSchedulerInterface scheduler = getScheduler(process.getSchedule().getSchedulerName()); - scheduler.setupProcess(process, backendVariantData, SchedulerUtils.allowedToStart(process)); + scheduler.setupProcess(process, backendVariantData, process.getSchedule(), SchedulerUtils.allowedToStart(process)); } @@ -269,8 +275,18 @@ public class QScheduleManager *******************************************************************************/ private void setupSqsProvider(SQSQueueProviderMetaData queueProvider) { - QSchedulerInterface scheduler = getScheduler(queueProvider.getSchedule().getSchedulerName()); - scheduler.setupSqsProvider(queueProvider, SchedulerUtils.allowedToStart(queueProvider)); + boolean allowedToStartProvider = SchedulerUtils.allowedToStart(queueProvider); + + for(QQueueMetaData queue : qInstance.getQueues().values()) + { + QSchedulerInterface scheduler = getScheduler(queue.getSchedule().getSchedulerName()); + + boolean allowedToStart = allowedToStartProvider && SchedulerUtils.allowedToStart(queue.getName()); + if(queueProvider.getName().equals(queue.getProviderName())) + { + scheduler.setupSqsPoller(queueProvider, queue, queue.getSchedule(), allowedToStart); + } + } } @@ -280,8 +296,21 @@ public class QScheduleManager *******************************************************************************/ private void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider) { - QSchedulerInterface scheduler = getScheduler(automationProvider.getSchedule().getSchedulerName()); - scheduler.setupAutomationProviderPerTable(automationProvider, SchedulerUtils.allowedToStart(automationProvider)); + boolean allowedToStartProvider = SchedulerUtils.allowedToStart(automationProvider); + + /////////////////////////////////////////////////////////////////////////////////// + // ask the PollingAutomationPerTableRunner how many threads of itself need setup // + // then schedule each one of them. // + /////////////////////////////////////////////////////////////////////////////////// + List tableActionList = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName()); + for(PollingAutomationPerTableRunner.TableActionsInterface tableActions : tableActionList) + { + boolean allowedToStart = allowedToStartProvider && SchedulerUtils.allowedToStart(tableActions.tableName()); + + QScheduleMetaData schedule = tableActions.tableAutomationDetails().getSchedule(); + QSchedulerInterface scheduler = getScheduler(schedule.getSchedulerName()); + scheduler.setupTableAutomation(automationProvider, tableActions, schedule, allowedToStart); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java index 87d9cea2..3e61c77f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java @@ -24,9 +24,12 @@ package com.kingsrook.qqq.backend.core.scheduler; import java.io.Serializable; import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; /******************************************************************************* @@ -37,17 +40,17 @@ public interface QSchedulerInterface /******************************************************************************* ** *******************************************************************************/ - void setupProcess(QProcessMetaData process, Map backendVariantData, boolean allowedToStart); + void setupProcess(QProcessMetaData process, Map backendVariantData, QScheduleMetaData schedule, boolean allowedToStart); /******************************************************************************* ** *******************************************************************************/ - void setupSqsProvider(SQSQueueProviderMetaData queueProvider, boolean allowedToStart); + void setupSqsPoller(SQSQueueProviderMetaData queueProvider, QQueueMetaData queue, QScheduleMetaData schedule, boolean allowedToStart); /******************************************************************************* ** *******************************************************************************/ - void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStart); + void setupTableAutomation(QAutomationProviderMetaData automationProvider, PollingAutomationPerTableRunner.TableActionsInterface tableActions, QScheduleMetaData schedule, boolean allowedToStart); /******************************************************************************* ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java index dee8cf9a..1e3189c7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java @@ -36,6 +36,7 @@ import java.util.Properties; import java.util.Set; import java.util.TimeZone; import java.util.function.Supplier; +import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -46,7 +47,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMeta import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; -import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; import com.kingsrook.qqq.backend.core.utils.memoization.AnyKey; import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; import org.quartz.CronExpression; @@ -83,12 +83,24 @@ public class QuartzScheduler implements QSchedulerInterface private Scheduler scheduler; + ///////////////////////////////////////////////////////////////////////////////////////// + // create memoization objects for some quartz-query functions, that we'll only want to // + // use during our setup routine, when we'd query it many times over and over again. // + // So default to a timeout of 0 (effectively disabling memoization). then in the // + // start-of-setup and end-of-setup methods, temporarily increase, then re-decrease // + ///////////////////////////////////////////////////////////////////////////////////////// private Memoization> jobGroupNamesMemoization = new Memoization>() - .withTimeout(Duration.of(5, ChronoUnit.SECONDS)); + .withTimeout(Duration.of(0, ChronoUnit.SECONDS)); private Memoization> jobKeyNamesMemoization = new Memoization>() - .withTimeout(Duration.of(5, ChronoUnit.SECONDS)); + .withTimeout(Duration.of(0, ChronoUnit.SECONDS)); + /////////////////////////////////////////////////////////////////////////////// + // vars used during the setup routine, to figure out what jobs need deleted. // + /////////////////////////////////////////////////////////////////////////////// + private boolean insideSetup = false; + private List scheduledJobsAtStartOfSetup = new ArrayList<>(); + private List scheduledJobsAtEndOfSetup = new ArrayList<>(); /******************************************************************************* @@ -202,7 +214,7 @@ public class QuartzScheduler implements QSchedulerInterface ** *******************************************************************************/ @Override - public void setupProcess(QProcessMetaData process, Map backendVariantData, boolean allowedToStart) + public void setupProcess(QProcessMetaData process, Map backendVariantData, QScheduleMetaData schedule, boolean allowedToStart) { ///////////////////////// // set up job data map // @@ -215,7 +227,72 @@ public class QuartzScheduler implements QSchedulerInterface jobData.put("backendVariantData", backendVariantData); } - scheduleJob(process.getName(), "processes", QuartzRunProcessJob.class, jobData, process.getSchedule(), allowedToStart); + scheduleJob(process.getName(), "processes", QuartzRunProcessJob.class, jobData, schedule, allowedToStart); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void startOfSetupSchedules() + { + this.insideSetup = true; + this.jobGroupNamesMemoization.setTimeout(Duration.ofSeconds(5)); + this.jobKeyNamesMemoization.setTimeout(Duration.ofSeconds(5)); + + try + { + this.scheduledJobsAtStartOfSetup = queryQuartz(); + } + catch(Exception e) + { + LOG.warn("Error querying quartz for the currently scheduled jobs during startup - will not be able to delete no-longer-needed jobs!", e); + } + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void endOfSetupSchedules() + { + this.insideSetup = false; + this.jobGroupNamesMemoization.setTimeout(Duration.ofSeconds(0)); + this.jobKeyNamesMemoization.setTimeout(Duration.ofSeconds(0)); + + if(this.scheduledJobsAtStartOfSetup == null) + { + return; + } + + try + { + Set startJobKeys = this.scheduledJobsAtStartOfSetup.stream().map(w -> w.jobDetail().getKey()).collect(Collectors.toSet()); + Set endJobKeys = scheduledJobsAtEndOfSetup.stream().map(w -> w.jobDetail().getKey()).collect(Collectors.toSet()); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // remove all 'end' keys from the set of start keys. any left-over start-keys need to be deleted. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + startJobKeys.removeAll(endJobKeys); + for(JobKey jobKey : startJobKeys) + { + LOG.info("Deleting job that had previously been scheduled, but doesn't appear to be any more", logPair("jobKey", jobKey)); + deleteJob(jobKey); + } + } + catch(Exception e) + { + LOG.warn("Error trying to clean up no-longer-needed jobs at end of scheduler setup", e); + } + + //////////////////////////////////////////////////// + // reset these lists, no need to keep them around // + //////////////////////////////////////////////////// + this.scheduledJobsAtStartOfSetup = null; + this.scheduledJobsAtEndOfSetup = null; } @@ -294,6 +371,15 @@ public class QuartzScheduler implements QSchedulerInterface resumeJob(jobKey.getName(), jobKey.getGroup()); } + /////////////////////////////////////////////////////////////////////////// + // if we're inside the setup event (e.g., initial startup), then capture // + // this job as one that is currently active and should be kept. // + /////////////////////////////////////////////////////////////////////////// + if(insideSetup) + { + scheduledJobsAtEndOfSetup.add(new QuartzJobAndTriggerWrapper(jobDetail, trigger, null)); + } + return (true); } catch(Exception e) @@ -309,57 +395,37 @@ public class QuartzScheduler implements QSchedulerInterface ** *******************************************************************************/ @Override - public void setupSqsProvider(SQSQueueProviderMetaData sqsQueueProvider, boolean allowedToStartProvider) + public void setupSqsPoller(SQSQueueProviderMetaData queueProvider, QQueueMetaData queue, QScheduleMetaData schedule, boolean allowedToStart) { - for(QQueueMetaData queue : qInstance.getQueues().values()) - { - boolean allowedToStart = allowedToStartProvider && SchedulerUtils.allowedToStart(queue.getName()); + ///////////////////////// + // set up job data map // + ///////////////////////// + Map jobData = new HashMap<>(); + jobData.put("queueProviderName", queueProvider.getName()); + jobData.put("queueName", queue.getName()); - if(sqsQueueProvider.getName().equals(queue.getProviderName())) - { - ///////////////////////// - // set up job data map // - ///////////////////////// - Map jobData = new HashMap<>(); - jobData.put("queueProviderName", sqsQueueProvider.getName()); - jobData.put("queueName", queue.getName()); - - scheduleJob(queue.getName(), "sqsQueue", QuartzSqsPollerJob.class, jobData, sqsQueueProvider.getSchedule(), allowedToStart); - } - } + scheduleJob(queue.getName(), "sqsQueue", QuartzSqsPollerJob.class, jobData, schedule, allowedToStart); } - /******************************************************************************* ** *******************************************************************************/ @Override - public void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStartProvider) + public void setupTableAutomation(QAutomationProviderMetaData automationProvider, PollingAutomationPerTableRunner.TableActionsInterface tableActions, QScheduleMetaData schedule, boolean allowedToStart) { - /////////////////////////////////////////////////////////////////////////////////// - // ask the PollingAutomationPerTableRunner how many threads of itself need setup // - // then start a scheduled executor foreach one // - /////////////////////////////////////////////////////////////////////////////////// - List tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName()); - for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions) - { - boolean allowedToStart = allowedToStartProvider && SchedulerUtils.allowedToStart(tableAction.tableName()); + ///////////////////////// + // set up job data map // + ///////////////////////// + Map jobData = new HashMap<>(); + jobData.put("automationProviderName", automationProvider.getName()); + jobData.put("tableName", tableActions.tableName()); + jobData.put("automationStatus", tableActions.status().toString()); - ///////////////////////// - // set up job data map // - ///////////////////////// - Map jobData = new HashMap<>(); - jobData.put("automationProviderName", automationProvider.getName()); - jobData.put("tableName", tableAction.tableName()); - jobData.put("automationStatus", tableAction.status().toString()); - - scheduleJob(tableAction.tableName() + "." + tableAction.status(), "tableAutomations", QuartzTableAutomationsJob.class, jobData, automationProvider.getSchedule(), allowedToStart); - } + scheduleJob(tableActions.tableName() + "." + tableActions.status(), "tableAutomations", QuartzTableAutomationsJob.class, jobData, schedule, allowedToStart); } - /******************************************************************************* ** *******************************************************************************/ @@ -416,7 +482,7 @@ public class QuartzScheduler implements QSchedulerInterface /******************************************************************************* ** *******************************************************************************/ - private boolean deleteJob(JobKey jobKey) + public boolean deleteJob(JobKey jobKey) { try { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java index 9d34bc22..c88fa495 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java @@ -27,6 +27,8 @@ import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomati import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; import org.quartz.JobDataMap; @@ -63,7 +65,24 @@ public class QuartzTableAutomationsJob implements Job automationStatus = AutomationStatus.valueOf(jobDataMap.getString("automationStatus")); QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); - PollingAutomationPerTableRunner.TableActionsInterface tableAction = new PollingAutomationPerTableRunner.TableActions(tableName, automationStatus); + QTableMetaData table = qInstance.getTable(tableName); + if(table == null) + { + LOG.warn("Could not find table for automations in QInstance", logPair("tableName", tableName)); + return; + } + + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + if(automationDetails == null) + { + LOG.warn("Could not find automationDetails for table for automations in QInstance", logPair("tableName", tableName)); + return; + } + + /////////////////////////////////// + // todo - sharded automations... // + /////////////////////////////////// + PollingAutomationPerTableRunner.TableActionsInterface tableAction = new PollingAutomationPerTableRunner.TableActions(tableName, automationDetails, automationStatus); PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProviderName, QuartzScheduler.getInstance().getSessionSupplier(), tableAction); ///////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java index e87e039a..af062d0b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java @@ -26,7 +26,6 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; @@ -141,32 +140,19 @@ public class SimpleScheduler implements QSchedulerInterface ** *******************************************************************************/ @Override - public void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider, boolean allowedToStartProvider) + public void setupTableAutomation(QAutomationProviderMetaData automationProvider, PollingAutomationPerTableRunner.TableActionsInterface tableActions, QScheduleMetaData schedule, boolean allowedToStart) { - if(!allowedToStartProvider) + if(!allowedToStart) { return; } - /////////////////////////////////////////////////////////////////////////////////// - // ask the PollingAutomationPerTableRunner how many threads of itself need setup // - // then start a scheduled executor foreach one // - /////////////////////////////////////////////////////////////////////////////////// - List tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName()); - for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions) - { - if(SchedulerUtils.allowedToStart(tableAction.tableName())) - { - PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProvider.getName(), sessionSupplier, tableAction); - StandardScheduledExecutor executor = new StandardScheduledExecutor(runner); + PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProvider.getName(), sessionSupplier, tableActions); + StandardScheduledExecutor executor = new StandardScheduledExecutor(runner); - QScheduleMetaData schedule = Objects.requireNonNullElseGet(automationProvider.getSchedule(), this::getDefaultSchedule); - - executor.setName(runner.getName()); - setScheduleInExecutor(schedule, executor); - executors.add(executor); - } - } + executor.setName(runner.getName()); + setScheduleInExecutor(schedule, executor); + executors.add(executor); } @@ -175,9 +161,9 @@ public class SimpleScheduler implements QSchedulerInterface ** *******************************************************************************/ @Override - public void setupSqsProvider(SQSQueueProviderMetaData queueProvider, boolean allowedToStartProvider) + public void setupSqsPoller(SQSQueueProviderMetaData queueProvider, QQueueMetaData queue, QScheduleMetaData schedule, boolean allowedToStart) { - if(!allowedToStartProvider) + if(!allowedToStart) { return; } @@ -185,27 +171,17 @@ public class SimpleScheduler implements QSchedulerInterface QInstance scheduleManagerQueueInstance = qInstance; Supplier scheduleManagerSessionSupplier = sessionSupplier; - for(QQueueMetaData queue : qInstance.getQueues().values()) - { - if(queueProvider.getName().equals(queue.getProviderName()) && SchedulerUtils.allowedToStart(queue.getName())) - { - SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); - sqsQueuePoller.setQueueProviderMetaData(queueProvider); - sqsQueuePoller.setQueueMetaData(queue); - sqsQueuePoller.setQInstance(scheduleManagerQueueInstance); - sqsQueuePoller.setSessionSupplier(scheduleManagerSessionSupplier); + SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); + sqsQueuePoller.setQueueProviderMetaData(queueProvider); + sqsQueuePoller.setQueueMetaData(queue); + sqsQueuePoller.setQInstance(scheduleManagerQueueInstance); + sqsQueuePoller.setSessionSupplier(scheduleManagerSessionSupplier); - StandardScheduledExecutor executor = new StandardScheduledExecutor(sqsQueuePoller); + StandardScheduledExecutor executor = new StandardScheduledExecutor(sqsQueuePoller); - QScheduleMetaData schedule = Objects.requireNonNullElseGet(queue.getSchedule(), - () -> Objects.requireNonNullElseGet(queueProvider.getSchedule(), - this::getDefaultSchedule)); - - executor.setName(queue.getName()); - setScheduleInExecutor(schedule, executor); - executors.add(executor); - } - } + executor.setName(queue.getName()); + setScheduleInExecutor(schedule, executor); + executors.add(executor); } @@ -214,7 +190,7 @@ public class SimpleScheduler implements QSchedulerInterface ** *******************************************************************************/ @Override - public void setupProcess(QProcessMetaData process, Map backendVariantData, boolean allowedToStart) + public void setupProcess(QProcessMetaData process, Map backendVariantData, QScheduleMetaData schedule, boolean allowedToStart) { if(!allowedToStart) { @@ -228,7 +204,7 @@ public class SimpleScheduler implements QSchedulerInterface StandardScheduledExecutor executor = new StandardScheduledExecutor(runProcess); executor.setName("process:" + process.getName()); - setScheduleInExecutor(process.getSchedule(), executor); + setScheduleInExecutor(schedule, executor); executors.add(executor); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java index abaea628..36c858f5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/StandardScheduledExecutor.java @@ -32,8 +32,10 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; /******************************************************************************* - ** Standard class ran by ScheduleManager. Takes a Runnable in its constructor - - ** that's the code that actually executes. + ** Standard class ran by SimpleScheduler. Takes a Runnable in its constructor - + ** that's the code that actually executes. Internally, this class will launch + ** a newSingleThreadScheduledExecutor / ScheduledExecutorService to run the + ** runnable on a repeating delay. ** *******************************************************************************/ public class StandardScheduledExecutor diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java index dc276b0d..c4e47381 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java @@ -575,8 +575,8 @@ class PollingAutomationPerTableRunnerTest extends BaseTest @Test void testLoadingRecordTypesToEnsureClassCoverage() { - new PollingAutomationPerTableRunner.TableActions(null, null).noopToFakeTestCoverage(); - new PollingAutomationPerTableRunner.ShardedTableActions(null, null, null, null, null).noopToFakeTestCoverage(); + new PollingAutomationPerTableRunner.TableActions(null, null, null).noopToFakeTestCoverage(); + new PollingAutomationPerTableRunner.ShardedTableActions(null, null, null, null, null, null).noopToFakeTestCoverage(); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 30b3a319..0ad55dde 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -469,19 +469,19 @@ public class QInstanceValidatorTest extends BaseTest assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("UTC"))); assertValidationSuccess((qInstance) -> qInstance.getProcess(processName).withSchedule(baseScheduleMetaData.get().withCronExpression(validCronString).withCronTimeZoneId("America/New_York"))); - ////////////////////////////////////////////////////////////////// - // make sure automation providers get their schedules validated // - ////////////////////////////////////////////////////////////////// - assertValidationFailureReasons((qInstance) -> qInstance.getAutomationProvider(TestUtils.POLLING_AUTOMATION).withSchedule(baseScheduleMetaData.get() + /////////////////////////////////////////////////////////////// + // make sure table automations get their schedules validated // + /////////////////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().withSchedule(baseScheduleMetaData.get() .withSchedulerName(null) .withCronExpression(validCronString) .withCronTimeZoneId("UTC")), "is missing a scheduler name"); - ///////////////////////////////////////////////////////////// - // make sure queue providers get their schedules validated // - ///////////////////////////////////////////////////////////// - assertValidationFailureReasons((qInstance) -> ((SQSQueueProviderMetaData)qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER)).withSchedule(baseScheduleMetaData.get() + //////////////////////////////////////////////////// + // make sure queues get their schedules validated // + //////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> (qInstance.getQueue(TestUtils.TEST_SQS_QUEUE)).withSchedule(baseScheduleMetaData.get() .withSchedulerName(null) .withCronExpression(validCronString) .withCronTimeZoneId("UTC")), diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java index a69e2e14..d3b3ba66 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java @@ -43,7 +43,9 @@ import com.kingsrook.qqq.backend.core.utils.SleepUtils; import org.apache.logging.log4j.Level; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; +import org.quartz.SchedulerException; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -59,7 +61,27 @@ class QuartzSchedulerTest extends BaseTest @AfterEach void afterEach() { - QScheduleManager.getInstance().unInit(); + try + { + QScheduleManager.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + + try + { + QuartzScheduler.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } } @@ -84,15 +106,7 @@ class QuartzSchedulerTest extends BaseTest ////////////////////////////////////////// // add a process we can run and observe // ////////////////////////////////////////// - qInstance.addProcess(new QProcessMetaData() - .withName("testScheduledProcess") - .withSchedule(new QScheduleMetaData() - .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) - .withRepeatMillis(2) - .withInitialDelaySeconds(0)) - .withStepList(List.of(new QBackendStepMetaData() - .withName("step") - .withCode(new QCodeReference(BasicStep.class))))); + qInstance.addProcess(buildTestProcess("testScheduledProcess")); ////////////////////////////////////////////////////////////////////////////// // start the schedule manager, which will schedule things, and start quartz // @@ -108,7 +122,7 @@ class QuartzSchedulerTest extends BaseTest qScheduleManager.stopAsync(); System.out.println("Ran: " + BasicStep.counter + " times"); - assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice (but only ran [" + BasicStep.counter + "] time(s)."); + assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice (but only ran [" + BasicStep.counter + "] time(s))."); ////////////////////////////////////////////////////// // make sure poller ran, and didn't issue any warns // @@ -132,6 +146,56 @@ class QuartzSchedulerTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + private static QProcessMetaData buildTestProcess(String name) + { + return new QProcessMetaData() + .withName(name) + .withSchedule(new QScheduleMetaData() + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withRepeatMillis(2) + .withInitialDelaySeconds(0)) + .withStepList(List.of(new QBackendStepMetaData() + .withName("step") + .withCode(new QCodeReference(BasicStep.class)))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRemovingNoLongerNeededJobsDuringSetupSchedules() throws SchedulerException + { + QInstance qInstance = QContext.getQInstance(); + QuartzTestUtils.setupInstanceForQuartzTests(); + + //////////////////////////// + // put two jobs in quartz // + //////////////////////////// + QProcessMetaData test1 = buildTestProcess("test1"); + QProcessMetaData test2 = buildTestProcess("test2"); + qInstance.addProcess(test1); + qInstance.addProcess(test2); + + QuartzScheduler quartzScheduler = QuartzScheduler.initInstance(qInstance, QuartzTestUtils.QUARTZ_SCHEDULER_NAME, QuartzTestUtils.getQuartzProperties(), () -> QContext.getQSession()); + quartzScheduler.setupProcess(test1, null, test1.getSchedule(), false); + quartzScheduler.setupProcess(test2, null, test2.getSchedule(), false); + + quartzScheduler.startOfSetupSchedules(); + quartzScheduler.setupProcess(test1, null, test1.getSchedule(), false); + quartzScheduler.endOfSetupSchedules(); + + List quartzJobAndTriggerWrappers = quartzScheduler.queryQuartz(); + assertEquals(1, quartzJobAndTriggerWrappers.size()); + assertEquals("test1", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java index 702207f3..44ba5437 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.Properties; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData; import org.quartz.SchedulerException; @@ -43,7 +42,7 @@ public class QuartzTestUtils /******************************************************************************* ** *******************************************************************************/ - private static Properties getQuartzProperties() + public static Properties getQuartzProperties() { Properties quartzProperties = new Properties(); quartzProperties.put("org.quartz.scheduler.instanceName", QUARTZ_SCHEDULER_NAME); @@ -76,12 +75,16 @@ public class QuartzTestUtils //////////////////////////////////////////////////////////////////////////////// // set the queue providers & automation providers to use the quartz scheduler // //////////////////////////////////////////////////////////////////////////////// - qInstance.getAutomationProviders().values() - .forEach(ap -> ap.getSchedule().setSchedulerName(QUARTZ_SCHEDULER_NAME)); - - qInstance.getQueueProviders().values() - .forEach(qp -> ((SQSQueueProviderMetaData) qp).getSchedule().setSchedulerName(QUARTZ_SCHEDULER_NAME)); + qInstance.getTables().values().forEach(t -> + { + if(t.getAutomationDetails() != null) + { + t.getAutomationDetails().getSchedule().setSchedulerName(QUARTZ_SCHEDULER_NAME); + } + }); + qInstance.getQueues().values() + .forEach(q -> q.getSchedule().setSchedulerName(QUARTZ_SCHEDULER_NAME)); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java index 760d12c3..743db22d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java @@ -89,8 +89,28 @@ class QuartzJobsProcessTest extends BaseTest @AfterEach void afterEach() { - QScheduleManager.getInstance().stop(); - QScheduleManager.getInstance().unInit(); + try + { + QScheduleManager.getInstance().stop(); + QScheduleManager.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + + try + { + QuartzScheduler.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index fde45605..0086c351 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -180,6 +180,7 @@ public class TestUtils public static final String SECURITY_KEY_TYPE_INTERNAL_OR_EXTERNAL = "internalOrExternal"; public static final String SIMPLE_SCHEDULER_NAME = "simpleScheduler"; + public static final String TEST_SQS_QUEUE = "testSQSQueue"; @@ -366,10 +367,7 @@ public class TestUtils private static QAutomationProviderMetaData definePollingAutomationProvider() { return (new PollingAutomationProviderMetaData() - .withName(POLLING_AUTOMATION) - .withSchedule(new QScheduleMetaData() - .withSchedulerName(SIMPLE_SCHEDULER_NAME) - .withRepeatSeconds(60))); + .withName(POLLING_AUTOMATION)); } @@ -746,6 +744,9 @@ public class TestUtils { return (new QTableAutomationDetails() .withProviderName(POLLING_AUTOMATION) + .withSchedule(new QScheduleMetaData() + .withSchedulerName(SIMPLE_SCHEDULER_NAME) + .withRepeatSeconds(60)) .withStatusTracking(new AutomationStatusTracking() .withType(AutomationStatusTrackingType.FIELD_IN_TABLE) .withFieldName("qqqAutomationStatus"))); @@ -1333,10 +1334,7 @@ public class TestUtils .withAccessKey(accessKey) .withSecretKey(secretKey) .withRegion(region) - .withBaseURL(baseURL) - .withSchedule(new QScheduleMetaData() - .withRepeatSeconds(60) - .withSchedulerName(SIMPLE_SCHEDULER_NAME))); + .withBaseURL(baseURL)); } @@ -1347,10 +1345,13 @@ public class TestUtils private static QQueueMetaData defineTestSqsQueue() { return (new QQueueMetaData() - .withName("testSQSQueue") + .withName(TEST_SQS_QUEUE) .withProviderName(DEFAULT_QUEUE_PROVIDER) .withQueueName("test-queue") - .withProcessName(PROCESS_NAME_INCREASE_BIRTHDATE)); + .withProcessName(PROCESS_NAME_INCREASE_BIRTHDATE) + .withSchedule(new QScheduleMetaData() + .withRepeatSeconds(60) + .withSchedulerName(SIMPLE_SCHEDULER_NAME))); } From 5f729785286b02cd08a7cb6f49c5ad02d082133b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 14 Mar 2024 11:14:27 -0500 Subject: [PATCH 38/72] Refactor table customizers to follow a common interface (point being, so you can have 1 class instead of many for a table's closely-related actions) --- .../AbstractPostDeleteCustomizer.java | 14 +- .../AbstractPostInsertCustomizer.java | 14 +- .../AbstractPostQueryCustomizer.java | 23 +- .../AbstractPostUpdateCustomizer.java | 16 +- .../AbstractPreDeleteCustomizer.java | 15 +- .../AbstractPreInsertCustomizer.java | 26 ++- .../AbstractPreUpdateCustomizer.java | 17 +- .../core/actions/customizers/QCodeLoader.java | 4 +- .../customizers/TableCustomizerInterface.java | 202 ++++++++++++++++++ .../actions/customizers/TableCustomizers.java | 14 +- .../core/actions/tables/DeleteAction.java | 14 +- .../core/actions/tables/GetAction.java | 18 +- .../core/actions/tables/InsertAction.java | 27 ++- .../core/actions/tables/QueryAction.java | 8 +- .../core/actions/tables/UpdateAction.java | 16 +- .../tables/QueryOrGetInputInterface.java | 5 + .../bulk/insert/BulkInsertTransformStep.java | 9 +- .../core/actions/tables/InsertActionTest.java | 144 +++++++++++++ .../actions/AbstractBaseFilesystemAction.java | 13 +- 19 files changed, 523 insertions(+), 76 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java index edf9a9cf..52864385 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java @@ -47,12 +47,24 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** records that the delete action marked in error - the user might want to do ** something special with them (idk, try some other way to delete them?) *******************************************************************************/ -public abstract class AbstractPostDeleteCustomizer +public abstract class AbstractPostDeleteCustomizer implements TableCustomizerInterface { protected DeleteInput deleteInput; + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postDelete(DeleteInput deleteInput, List records) throws QException + { + this.deleteInput = deleteInput; + return apply(records); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java index c7e2bfc6..100fe267 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java @@ -42,12 +42,24 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** ** Note that the full insertInput is available as a field in this class. *******************************************************************************/ -public abstract class AbstractPostInsertCustomizer +public abstract class AbstractPostInsertCustomizer implements TableCustomizerInterface { protected InsertInput insertInput; + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + this.insertInput = insertInput; + return (apply(records)); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java index d1beaa4c..669aa06b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java @@ -23,16 +23,29 @@ package com.kingsrook.qqq.backend.core.actions.customizers; import java.util.List; -import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; import com.kingsrook.qqq.backend.core.model.data.QRecord; /******************************************************************************* ** *******************************************************************************/ -public abstract class AbstractPostQueryCustomizer +public abstract class AbstractPostQueryCustomizer implements TableCustomizerInterface { - protected AbstractTableActionInput input; + protected QueryOrGetInputInterface input; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postQuery(QueryOrGetInputInterface queryInput, List records) throws QException + { + input = queryInput; + return apply(records); + } @@ -47,7 +60,7 @@ public abstract class AbstractPostQueryCustomizer ** Getter for input ** *******************************************************************************/ - public AbstractTableActionInput getInput() + public QueryOrGetInputInterface getInput() { return (input); } @@ -58,7 +71,7 @@ public abstract class AbstractPostQueryCustomizer ** Setter for input ** *******************************************************************************/ - public void setInput(AbstractTableActionInput input) + public void setInput(QueryOrGetInputInterface input) { this.input = input; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java index b0d55b35..53e00583 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -48,7 +49,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** available (if the backend supports it) - both as a list (`getOldRecordList`) ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`). *******************************************************************************/ -public abstract class AbstractPostUpdateCustomizer +public abstract class AbstractPostUpdateCustomizer implements TableCustomizerInterface { protected UpdateInput updateInput; protected List oldRecordList; @@ -57,6 +58,19 @@ public abstract class AbstractPostUpdateCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + this.updateInput = updateInput; + this.oldRecordList = oldRecordList.orElse(null); + return apply(records); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java index 4b848a14..80460a86 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java @@ -50,7 +50,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** Note that the full deleteInput is available as a field in this class. ** *******************************************************************************/ -public abstract class AbstractPreDeleteCustomizer +public abstract class AbstractPreDeleteCustomizer implements TableCustomizerInterface { protected DeleteInput deleteInput; @@ -58,6 +58,19 @@ public abstract class AbstractPreDeleteCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preDelete(DeleteInput deleteInput, List records, boolean isPreview) throws QException + { + this.deleteInput = deleteInput; + this.isPreview = isPreview; + return apply(records); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java index 196ea4b8..c0706a5c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java @@ -47,7 +47,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** ** Note that the full insertInput is available as a field in this class. *******************************************************************************/ -public abstract class AbstractPreInsertCustomizer +public abstract class AbstractPreInsertCustomizer implements TableCustomizerInterface { protected InsertInput insertInput; @@ -70,6 +70,30 @@ public abstract class AbstractPreInsertCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + this.insertInput = insertInput; + this.isPreview = isPreview; + return (apply(records)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview) + { + return getWhenToRun(); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java index b8a95ed2..701ce30c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -53,7 +54,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** available (if the backend supports it) - both as a list (`getOldRecordList`) ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`). *******************************************************************************/ -public abstract class AbstractPreUpdateCustomizer +public abstract class AbstractPreUpdateCustomizer implements TableCustomizerInterface { protected UpdateInput updateInput; protected List oldRecordList; @@ -63,6 +64,20 @@ public abstract class AbstractPreUpdateCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + this.updateInput = updateInput; + this.isPreview = isPreview; + this.oldRecordList = oldRecordList.orElse(null); + return apply(records); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java index 05acf79a..23162753 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java @@ -64,12 +64,12 @@ public class QCodeLoader /******************************************************************************* ** *******************************************************************************/ - public static Optional getTableCustomizer(Class expectedClass, QTableMetaData table, String customizerName) + public static Optional getTableCustomizer(QTableMetaData table, String customizerName) { Optional codeReference = table.getCustomizer(customizerName); if(codeReference.isPresent()) { - return (Optional.ofNullable(QCodeLoader.getAdHoc(expectedClass, codeReference.get()))); + return (Optional.ofNullable(QCodeLoader.getAdHoc(TableCustomizerInterface.class, codeReference.get()))); } return (Optional.empty()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java new file mode 100644 index 00000000..3a7cfa41 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java @@ -0,0 +1,202 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.customizers; + + +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Common interface used by all (core) TableCustomizer types (e.g., post-query, + ** and {pre,post}-{insert,update,delete}. + ** + ** Note that the abstract-base classes for each action still exist, though have + ** been back-ported to be implementors of this interface. The action classes + ** will now expect this type, and call this type's methods. + ** + *******************************************************************************/ +public interface TableCustomizerInterface +{ + QLogger LOG = QLogger.getLogger(TableCustomizerInterface.class); + + + /******************************************************************************* + ** custom actions to run after a query (or get!) takes place. + ** + *******************************************************************************/ + default List postQuery(QueryOrGetInputInterface queryInput, List records) throws QException + { + LOG.info("A default implementation of postQuery is running... Probably not expected!", logPair("tableName", queryInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** custom actions before an insert takes place. + ** + ** It's important for implementations to be aware of the isPreview field, which + ** is set to true when the code is running to give users advice, e.g., on a review + ** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing + ** things like storing data, you don't want to do that if isPreview is true!! + ** + ** General implementation would be, to iterate over the records (the inputs to + ** the insert action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records + ** - possibly manipulating values (`setValue`) + ** - possibly throwing an exception - if you really don't want the insert operation to continue. + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class. + *******************************************************************************/ + default List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + LOG.info("A default implementation of preInsert is running... Probably not expected!", logPair("tableName", insertInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + default AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview) + { + return (AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS); + } + + + /******************************************************************************* + ** custom actions after an insert takes place. + ** + ** General implementation would be, to iterate over the records (the outputs of + ** the insert action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records + ** - possibly throwing an exception - though doing so won't stop the update, and instead + ** will just set a warning on all of the updated records... + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go back to the caller. + *******************************************************************************/ + default List postInsert(InsertInput insertInput, List records) throws QException + { + LOG.info("A default implementation of postInsert is running... Probably not expected!", logPair("tableName", insertInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** custom actions before an update takes place. + ** + ** It's important for implementations to be aware of the isPreview field, which + ** is set to true when the code is running to give users advice, e.g., on a review + ** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing + ** things like storing data, you don't want to do that if isPreview is true!! + ** + ** General implementation would be, to iterate over the records (the inputs to + ** the update action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records + ** - possibly manipulating values (`setValue`) + ** - possibly throwing an exception - if you really don't want the update operation to continue. + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class. + ** + ** Note, "old records" (e.g., with values freshly fetched from the backend) will be + ** available (if the backend supports it) + *******************************************************************************/ + default List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + LOG.info("A default implementation of preUpdate is running... Probably not expected!", logPair("tableName", updateInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** custom actions after an update takes place. + ** + ** General implementation would be, to iterate over the records (the outputs of + ** the update action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records? + ** - possibly throwing an exception - though doing so won't stop the update, and instead + ** will just set a warning on all of the updated records... + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go back to the caller. + ** + ** Note, "old records" (e.g., with values freshly fetched from the backend) will be + ** available (if the backend supports it). + *******************************************************************************/ + default List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + LOG.info("A default implementation of postUpdate is running... Probably not expected!", logPair("tableName", updateInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** Custom actions before a delete takes place. + ** + ** It's important for implementations to be aware of the isPreview param, which + ** is set to true when the code is running to give users advice, e.g., on a review + ** screen - vs. being false when the action is ACTUALLY happening. So, if you're doing + ** things like storing data, you don't want to do that if isPreview is true!! + ** + ** General implementation would be, to iterate over the records (which the DeleteAction + ** would look up based on the inputs to the delete action), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records + ** - possibly throwing an exception - if you really don't want the delete operation to continue. + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) - this is how errors + ** and warnings are propagated to the DeleteAction. Note that any records with + ** an error will NOT proceed to the backend's delete interface - but those with + ** warnings will. + *******************************************************************************/ + default List preDelete(DeleteInput deleteInput, List records, boolean isPreview) throws QException + { + LOG.info("A default implementation of preDelete is running... Probably not expected!", logPair("tableName", deleteInput.getTableName())); + return (records); + } + + + /******************************************************************************* + ** Custom actions after a delete takes place. + ** + ** General implementation would be, to iterate over the records (ones which didn't + ** have a delete error), and look at their values: + ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records? + ** - possibly throwing an exception - though doing so won't stop the delete, and instead + ** will just set a warning on all of the deleted records... + ** - doing "whatever else" you may want to do. + ** - returning the list of records (can be the input list) that you want to go back + ** to the caller - this is how errors and warnings are propagated . + *******************************************************************************/ + default List postDelete(DeleteInput deleteInput, List records) throws QException + { + LOG.info("A default implementation of postDelete is running... Probably not expected!", logPair("tableName", deleteInput.getTableName())); + return (records); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java index 2c753b56..4c4d0f8d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java @@ -29,13 +29,13 @@ package com.kingsrook.qqq.backend.core.actions.customizers; *******************************************************************************/ public enum TableCustomizers { - POST_QUERY_RECORD("postQueryRecord", AbstractPostQueryCustomizer.class), - PRE_INSERT_RECORD("preInsertRecord", AbstractPreInsertCustomizer.class), - POST_INSERT_RECORD("postInsertRecord", AbstractPostInsertCustomizer.class), - PRE_UPDATE_RECORD("preUpdateRecord", AbstractPreUpdateCustomizer.class), - POST_UPDATE_RECORD("postUpdateRecord", AbstractPostUpdateCustomizer.class), - PRE_DELETE_RECORD("preDeleteRecord", AbstractPreDeleteCustomizer.class), - POST_DELETE_RECORD("postDeleteRecord", AbstractPostDeleteCustomizer.class); + POST_QUERY_RECORD("postQueryRecord", TableCustomizerInterface.class), + PRE_INSERT_RECORD("preInsertRecord", TableCustomizerInterface.class), + POST_INSERT_RECORD("postInsertRecord", TableCustomizerInterface.class), + PRE_UPDATE_RECORD("preUpdateRecord", TableCustomizerInterface.class), + POST_UPDATE_RECORD("postUpdateRecord", TableCustomizerInterface.class), + PRE_DELETE_RECORD("preDeleteRecord", TableCustomizerInterface.class), + POST_DELETE_RECORD("postDeleteRecord", TableCustomizerInterface.class); private final String role; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java index ee49499a..05cc83b1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java @@ -35,9 +35,8 @@ import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostDeleteCustomizer; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreDeleteCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper; @@ -250,7 +249,7 @@ public class DeleteAction ////////////////////////////////////////////////////////////// // finally, run the post-delete customizer, if there is one // ////////////////////////////////////////////////////////////// - Optional postDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPostDeleteCustomizer.class, table, TableCustomizers.POST_DELETE_RECORD.getRole()); + Optional postDeleteCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_DELETE_RECORD.getRole()); if(postDeleteCustomizer.isPresent() && oldRecordList.isPresent()) { //////////////////////////////////////////////////////////////////////////// @@ -260,8 +259,7 @@ public class DeleteAction try { - postDeleteCustomizer.get().setDeleteInput(deleteInput); - List postCustomizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer); + List postCustomizerResult = postDeleteCustomizer.get().postDelete(deleteInput, recordsForCustomizer); /////////////////////////////////////////////////////// // check if any records got errors in the customizer // @@ -327,13 +325,11 @@ public class DeleteAction /////////////////////////////////////////////////////////////////////////// // after all validations, run the pre-delete customizer, if there is one // /////////////////////////////////////////////////////////////////////////// - Optional preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole()); + Optional preDeleteCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_DELETE_RECORD.getRole()); List customizerResult = oldRecordList.get(); if(preDeleteCustomizer.isPresent()) { - preDeleteCustomizer.get().setDeleteInput(deleteInput); - preDeleteCustomizer.get().setIsPreview(isPreview); - customizerResult = preDeleteCustomizer.get().apply(oldRecordList.get()); + customizerResult = preDeleteCustomizer.get().preDelete(deleteInput, oldRecordList.get(), isPreview); } ///////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java index 8c19de56..bd110c32 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java @@ -27,8 +27,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.ActionHelper; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface; import com.kingsrook.qqq.backend.core.actions.tables.helpers.GetActionCacheHelper; @@ -58,7 +58,7 @@ import com.kingsrook.qqq.backend.core.utils.ObjectUtils; *******************************************************************************/ public class GetAction { - private Optional postGetRecordCustomizer; + private Optional postGetRecordCustomizer; private GetInput getInput; private QPossibleValueTranslator qPossibleValueTranslator; @@ -88,7 +88,7 @@ public class GetAction throw (new QException("Requested to Get a record from an unrecognized table: " + getInput.getTableName())); } - postGetRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole()); + postGetRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole()); this.getInput = getInput; QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); @@ -126,10 +126,10 @@ public class GetAction new GetActionCacheHelper().handleCaching(getInput, getOutput); } - /////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // if the record is found, perform post-actions on it // - // unless the defaultGetInteface was used - as it just does a query, and the query will do the post-actions. // - /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the record is found, perform post-actions on it // + // unless the defaultGetInterface was used - as it just does a query, and the query will do the post-actions. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// if(getOutput.getRecord() != null && !usingDefaultGetInterface) { getOutput.setRecord(postRecordActions(getOutput.getRecord())); @@ -220,12 +220,12 @@ public class GetAction ** Run the necessary actions on a record. This may include setting display values, ** translating possible values, and running post-record customizations. *******************************************************************************/ - public QRecord postRecordActions(QRecord record) + public QRecord postRecordActions(QRecord record) throws QException { QRecord returnRecord = record; if(this.postGetRecordCustomizer.isPresent()) { - returnRecord = postGetRecordCustomizer.get().apply(List.of(record)).get(0); + returnRecord = postGetRecordCustomizer.get().postQuery(getInput, List.of(record)).get(0); } if(getInput.getShouldTranslatePossibleValues()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 7e2c5dfb..9215c861 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -38,9 +38,9 @@ import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; @@ -169,13 +169,12 @@ public class InsertAction extends AbstractQActionFunction postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole()); + Optional postInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_INSERT_RECORD.getRole()); if(postInsertCustomizer.isPresent()) { try { - postInsertCustomizer.get().setInsertInput(insertInput); - insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords())); + insertOutput.setRecords(postInsertCustomizer.get().postInsert(insertInput, insertOutput.getRecords())); } catch(Exception e) { @@ -233,31 +232,29 @@ public class InsertAction extends AbstractQActionFunction preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole()); + Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); if(preInsertCustomizer.isPresent()) { - preInsertCustomizer.get().setInsertInput(insertInput); - preInsertCustomizer.get().setIsPreview(isPreview); - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS); } setDefaultValuesInRecords(table, insertInput.getRecords()); ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords(), null); - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS); setErrorsIfUniqueKeyErrors(insertInput, table); - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS); if(insertInput.getInputSource().shouldValidateRequiredFields()) { validateRequiredFields(insertInput); } - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS); ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT); - runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS); } @@ -291,13 +288,13 @@ public class InsertAction extends AbstractQActionFunction preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException + private void runPreInsertCustomizerIfItIsTime(InsertInput insertInput, boolean isPreview, Optional preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException { if(preInsertCustomizer.isPresent()) { - if(whenToRun.equals(preInsertCustomizer.get().getWhenToRun())) + if(whenToRun.equals(preInsertCustomizer.get().whenToRunPreInsert(insertInput, isPreview))) { - insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords())); + insertInput.setRecords(preInsertCustomizer.get().preInsert(insertInput, insertInput.getRecords(), isPreview)); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java index 3834edfb..09044873 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java @@ -31,8 +31,8 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.ActionHelper; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe; @@ -73,7 +73,7 @@ public class QueryAction { private static final QLogger LOG = QLogger.getLogger(QueryAction.class); - private Optional postQueryRecordCustomizer; + private Optional postQueryRecordCustomizer; private QueryInput queryInput; private QueryInterface queryInterface; @@ -100,7 +100,7 @@ public class QueryAction } QBackendMetaData backend = queryInput.getBackend(); - postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole()); + postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole()); this.queryInput = queryInput; if(queryInput.getRecordPipe() != null) @@ -264,7 +264,7 @@ public class QueryAction { if(this.postQueryRecordCustomizer.isPresent()) { - records = postQueryRecordCustomizer.get().apply(records); + records = postQueryRecordCustomizer.get().postQuery(queryInput, records); } if(queryInput.getShouldTranslatePossibleValues()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java index a69acc45..6a92cf80 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java @@ -34,9 +34,8 @@ import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostUpdateCustomizer; -import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreUpdateCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper; @@ -192,14 +191,12 @@ public class UpdateAction ////////////////////////////////////////////////////////////// // finally, run the post-update customizer, if there is one // ////////////////////////////////////////////////////////////// - Optional postUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPostUpdateCustomizer.class, table, TableCustomizers.POST_UPDATE_RECORD.getRole()); + Optional postUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_UPDATE_RECORD.getRole()); if(postUpdateCustomizer.isPresent()) { try { - postUpdateCustomizer.get().setUpdateInput(updateInput); - oldRecordList.ifPresent(l -> postUpdateCustomizer.get().setOldRecordList(l)); - updateOutput.setRecords(postUpdateCustomizer.get().apply(updateOutput.getRecords())); + updateOutput.setRecords(postUpdateCustomizer.get().postUpdate(updateInput, updateOutput.getRecords(), oldRecordList)); } catch(Exception e) { @@ -273,13 +270,10 @@ public class UpdateAction /////////////////////////////////////////////////////////////////////////// // after all validations, run the pre-update customizer, if there is one // /////////////////////////////////////////////////////////////////////////// - Optional preUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPreUpdateCustomizer.class, table, TableCustomizers.PRE_UPDATE_RECORD.getRole()); + Optional preUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_UPDATE_RECORD.getRole()); if(preUpdateCustomizer.isPresent()) { - preUpdateCustomizer.get().setUpdateInput(updateInput); - preUpdateCustomizer.get().setIsPreview(isPreview); - oldRecordList.ifPresent(l -> preUpdateCustomizer.get().setOldRecordList(l)); - updateInput.setRecords(preUpdateCustomizer.get().apply(updateInput.getRecords())); + updateInput.setRecords(preUpdateCustomizer.get().preUpdate(updateInput, updateInput.getRecords(), isPreview, oldRecordList)); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java index 42804602..cc361583 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java @@ -60,6 +60,11 @@ public interface QueryOrGetInputInterface QBackendTransaction getTransaction(); + /******************************************************************************* + ** + *******************************************************************************/ + String getTableName(); + /******************************************************************************* ** Setter for transaction *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 7f33a624..ae80f069 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -34,6 +34,7 @@ import java.util.Set; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; @@ -137,15 +138,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep // we do this, in case it needs to, for example, adjust values that // // are part of a unique key // ////////////////////////////////////////////////////////////////////// - Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole()); + Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); if(preInsertCustomizer.isPresent()) { - preInsertCustomizer.get().setInsertInput(insertInput); - preInsertCustomizer.get().setIsPreview(true); - AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().getWhenToRun(); + AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true); if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun)) { - List recordsAfterCustomizer = preInsertCustomizer.get().apply(runBackendStepInput.getRecords()); + List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, runBackendStepInput.getRecords(), true); runBackendStepInput.setRecords(recordsAfterCustomizer); /////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java index e17d6ed0..bbb5493b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java @@ -26,15 +26,21 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; @@ -777,4 +783,142 @@ class InsertActionTest extends BaseTest assertEquals(2, records.get(1).getValueInteger("noOfShoes")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCustomizers() throws QException + { + String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY; + + { + QContext.getQInstance().getTable(tableName).withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(TestPreInsertCustomizer.class)); + + List records = new InsertAction().execute(new InsertInput(tableName) + .withRecord(new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff"))) + .getRecords(); + assertEquals(1701, records.get(0).getValueInteger("noOfShoes")); + + /////////////////////////////////////////////////////////////////////////////////////////////////// + // because this was a pre-action, the value should actually be inserted - so re-query and get it // + /////////////////////////////////////////////////////////////////////////////////////////////////// + assertEquals(1701, new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(1)).getValueInteger("noOfShoes")); + + QContext.getQInstance().getTable(tableName).withCustomizers(new HashMap<>()); + } + + { + QContext.getQInstance().getTable(tableName).withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(TestPostInsertCustomizer.class)); + + List records = new InsertAction().execute(new InsertInput(tableName) + .withRecord(new QRecord().withValue("firstName", "Thom").withValue("lastName", "Chutterloin"))) + .getRecords(); + assertEquals(47, records.get(0).getValueInteger("homeStateId")); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // because this was a post-action, the value should NOT actually be inserted - so re-query and confirm null // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertNull(new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(2)).getValueInteger("homeStateId")); + + QContext.getQInstance().getTable(tableName).withCustomizers(new HashMap<>()); + } + + { + QContext.getQInstance().getTable(tableName).withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(TestTableCustomizer.class)); + QContext.getQInstance().getTable(tableName).withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(TestTableCustomizer.class)); + + List records = new InsertAction().execute(new InsertInput(tableName) + .withRecord(new QRecord().withValue("firstName", "Thom").withValue("lastName", "Chutterloin"))) + .getRecords(); + assertEquals(1701, records.get(0).getValueInteger("noOfShoes")); + assertEquals(47, records.get(0).getValueInteger("homeStateId")); + + ////////////////////////////////////////////////////////////////////// + // merger of the two above - one pre, one post, so one set, one not // + ////////////////////////////////////////////////////////////////////// + QRecord fetchedRecord = new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(2)); + assertEquals(1701, records.get(0).getValueInteger("noOfShoes")); + assertNull(fetchedRecord.getValueInteger("homeStateId")); + + QContext.getQInstance().getTable(tableName).withCustomizers(new HashMap<>()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestPreInsertCustomizer extends AbstractPreInsertCustomizer + { + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List apply(List records) throws QException + { + List rs = new ArrayList<>(); + records.forEach(r -> rs.add(new QRecord(r).withValue("noOfShoes", 1701))); + return rs; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestPostInsertCustomizer extends AbstractPostInsertCustomizer + { + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List apply(List records) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // grr, memory backend let's make sure to return a clone (so we don't edit what's stored!) // + ///////////////////////////////////////////////////////////////////////////////////////////// + List rs = new ArrayList<>(); + records.forEach(r -> rs.add(new QRecord(r).withValue("homeStateId", 47))); + return rs; + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestTableCustomizer implements TableCustomizerInterface + { + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + List rs = new ArrayList<>(); + records.forEach(r -> rs.add(new QRecord(r).withValue("noOfShoes", 1701))); + return rs; + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // grr, memory backend let's make sure to return a clone (so we don't edit what's stored!) // + ///////////////////////////////////////////////////////////////////////////////////////////// + List rs = new ArrayList<>(); + records.forEach(r -> rs.add(new QRecord(r).withValue("homeStateId", 47))); + return rs; + } + } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index df54eef9..ea07888a 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -43,6 +43,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -368,13 +369,19 @@ public abstract class AbstractBaseFilesystemAction { try { - Optional tableCustomizer = QCodeLoader.getTableCustomizer(AbstractPostReadFileCustomizer.class, table, FilesystemTableCustomizers.POST_READ_FILE.getRole()); - if(tableCustomizer.isEmpty()) + Optional codeReference = table.getCustomizer(FilesystemTableCustomizers.POST_READ_FILE.getRole()); + if(codeReference.isEmpty()) { return (fileContents); } - return tableCustomizer.get().customizeFileContents(fileContents); + AbstractPostReadFileCustomizer tableCustomizer = QCodeLoader.getAdHoc(AbstractPostReadFileCustomizer.class, codeReference.get()); + if(tableCustomizer == null) + { + return (fileContents); + } + + return tableCustomizer.customizeFileContents(fileContents); } catch(Exception e) { From 1a1ebcbe02fb4292e01a5154f121dedc996924c5 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 14 Mar 2024 11:14:45 -0500 Subject: [PATCH 39/72] Add constructor that takes field --- .../core/model/actions/tables/aggregate/GroupBy.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java index 750fc91b..87acc4a0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/GroupBy.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.aggregate; import java.io.Serializable; import java.util.Objects; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; @@ -38,6 +39,17 @@ public class GroupBy implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + public GroupBy(QFieldMetaData field) + { + this.type = field.getType(); + this.fieldName = field.getName(); + } + + + /******************************************************************************* ** *******************************************************************************/ From 2a97598309038b62501552d59d12982c7fabbbbf Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 14 Mar 2024 11:15:10 -0500 Subject: [PATCH 40/72] Put icons on these apps --- .../core/model/querystats/QueryStatMetaDataProvider.java | 2 ++ .../automation/HealBadRecordAutomationStatusesProcessStep.java | 2 ++ .../automation/RunTableAutomationsProcessStep.java | 2 ++ 3 files changed, 6 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java index 62e460bf..c9cbd826 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java @@ -65,6 +65,7 @@ public class QueryStatMetaDataProvider instance.addTable(defineStandardTable(QueryStatJoinTable.TABLE_NAME, QueryStatJoinTable.class, backendName, backendDetailEnricher)); instance.addTable(defineStandardTable(QueryStatCriteriaField.TABLE_NAME, QueryStatCriteriaField.class, backendName, backendDetailEnricher) + .withIcon(new QIcon().withName("filter_alt")) .withExposedJoin(new ExposedJoin().withJoinTable(QueryStat.TABLE_NAME)) ); @@ -115,6 +116,7 @@ public class QueryStatMetaDataProvider QTableMetaData table = new QTableMetaData() .withName(QueryStat.TABLE_NAME) + .withIcon(new QIcon().withName("query_stats")) .withBackendName(backendName) .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE)) .withRecordLabelFormat("%s") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java index 84b3ab31..310fafe6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java @@ -55,6 +55,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; @@ -129,6 +130,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, QProcessMetaData processMetaData = new QProcessMetaData() .withName(NAME) + .withIcon(new QIcon().withName("healing")) .withStepList(List.of( new QFrontendStepMetaData() .withName("input") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java index 1c02a563..739619dd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProvi import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; @@ -71,6 +72,7 @@ public class RunTableAutomationsProcessStep implements BackendStep, MetaDataProd { QProcessMetaData processMetaData = new QProcessMetaData() .withName(NAME) + .withIcon(new QIcon().withName("directions_run")) .withStepList(List.of( new QFrontendStepMetaData() .withName("input") From 60ffac4646249d1ad3d140bec522eb7a53ec8ade Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 14 Mar 2024 11:15:54 -0500 Subject: [PATCH 41/72] CE-936 - add reviewStepRecordFields to these processes --- .../quartz/processes/PauseQuartzJobsProcess.java | 9 ++++++++- .../quartz/processes/ResumeQuartzJobsProcess.java | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java index 96f033cd..f74791cc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java @@ -22,12 +22,15 @@ package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; +import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; @@ -61,6 +64,10 @@ public class PauseQuartzJobsProcess extends AbstractLoadStep implements MetaData .withTransformStepClass(NoopTransformStep.class) .withLoadStepClass(getClass()) .withIcon(new QIcon("pause_circle_outline")) + .withReviewStepRecordFields(List.of( + new QFieldMetaData("id", QFieldType.LONG), + new QFieldMetaData("jobName", QFieldType.STRING), + new QFieldMetaData("jobGroup", QFieldType.STRING))) .getProcessMetaData(); } @@ -77,7 +84,7 @@ public class PauseQuartzJobsProcess extends AbstractLoadStep implements MetaData QuartzScheduler instance = QuartzScheduler.getInstance(); for(QRecord record : runBackendStepInput.getRecords()) { - instance.pauseJob(record.getValueString("jobName"), record.getValueString("groupName")); + instance.pauseJob(record.getValueString("jobName"), record.getValueString("jobGroup")); } } catch(Exception e) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java index e7215452..687f29b3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java @@ -22,12 +22,15 @@ package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; +import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; @@ -61,6 +64,10 @@ public class ResumeQuartzJobsProcess extends AbstractLoadStep implements MetaDat .withTransformStepClass(NoopTransformStep.class) .withLoadStepClass(getClass()) .withIcon(new QIcon("play_circle_outline")) + .withReviewStepRecordFields(List.of( + new QFieldMetaData("id", QFieldType.LONG), + new QFieldMetaData("jobName", QFieldType.STRING), + new QFieldMetaData("jobGroup", QFieldType.STRING))) .getProcessMetaData(); } @@ -77,7 +84,7 @@ public class ResumeQuartzJobsProcess extends AbstractLoadStep implements MetaDat QuartzScheduler instance = QuartzScheduler.getInstance(); for(QRecord record : runBackendStepInput.getRecords()) { - instance.resumeJob(record.getValueString("jobName"), record.getValueString("groupName")); + instance.resumeJob(record.getValueString("jobName"), record.getValueString("jobGroup")); } } catch(Exception e) From 2fec4891d39268d474b2053db5bcb3703663d305 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 14 Mar 2024 11:18:18 -0500 Subject: [PATCH 42/72] CE-936 - dynamic scheduling of records from ScheduledJob table; quartz re-pause if paused before rescheduling --- .../actions/tables/query/QueryOutput.java | 16 + ...nePossibleValueSourceMetaDataProvider.java | 78 +++ .../model/scheduledjobs/ScheduledJob.java | 461 ++++++++++++++++++ .../scheduledjobs/ScheduledJobParameter.java | 266 ++++++++++ .../model/scheduledjobs/ScheduledJobType.java | 123 +++++ .../ScheduledJobsMetaDataProvider.java | 219 +++++++++ .../SchedulersPossibleValueSource.java | 87 ++++ .../ScheduledJobTableCustomizer.java | 278 +++++++++++ .../core/scheduler/QScheduleManager.java | 218 +++++++-- .../core/scheduler/QSchedulerInterface.java | 6 +- .../scheduler/quartz/QuartzScheduler.java | 129 +++-- ... => QuartzJobDataPostQueryCustomizer.java} | 13 +- .../scheduler/simple/SimpleScheduler.java | 12 + .../processes/QuartzJobsProcessTest.java | 4 +- 14 files changed, 1815 insertions(+), 95 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/common/TimeZonePossibleValueSourceMetaDataProvider.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobParameter.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/{QuartzJobDetailsPostQueryCustomizer.java => QuartzJobDataPostQueryCustomizer.java} (86%) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java index da9dad45..bbee71d3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java @@ -23,10 +23,12 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.io.Serializable; +import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; /******************************************************************************* @@ -89,4 +91,18 @@ public class QueryOutput extends AbstractActionOutput implements Serializable return storage.getRecords(); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public List getRecordEntities(Class entityClass) throws QException + { + List rs = new ArrayList<>(); + for(QRecord record : storage.getRecords()) + { + rs.add(QRecordEntity.fromQRecord(entityClass, record)); + } + return (rs); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/common/TimeZonePossibleValueSourceMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/common/TimeZonePossibleValueSourceMetaDataProvider.java new file mode 100644 index 00000000..fa06e309 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/common/TimeZonePossibleValueSourceMetaDataProvider.java @@ -0,0 +1,78 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.common; + + +import java.util.ArrayList; +import java.util.List; +import java.util.TimeZone; +import java.util.function.Function; +import java.util.function.Predicate; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TimeZonePossibleValueSourceMetaDataProvider +{ + public static final String NAME = "timeZones"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QPossibleValueSource produce() + { + return (produce(null, null)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QPossibleValueSource produce(Predicate filter, Function labelMapper) + { + QPossibleValueSource possibleValueSource = new QPossibleValueSource() + .withName("timeZones") + .withType(QPossibleValueSourceType.ENUM) + .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY); + + List> enumValues = new ArrayList<>(); + for(String availableID : TimeZone.getAvailableIDs()) + { + if(filter == null || filter.test(availableID)) + { + String label = labelMapper == null ? availableID : labelMapper.apply(availableID); + enumValues.add(new QPossibleValue<>(availableID, label)); + } + } + + possibleValueSource.withEnumValues(enumValues); + return (possibleValueSource); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java new file mode 100644 index 00000000..487252b4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java @@ -0,0 +1,461 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.common.TimeZonePossibleValueSourceMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.data.QAssociation; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MutableMap; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJob extends QRecordEntity +{ + public static final String TABLE_NAME = "scheduledJob"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(isRequired = true, maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE) + private String label; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS) + private String description; + + @QField(isRequired = true, label = "Scheduler", maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = SchedulersPossibleValueSource.NAME) + private String schedulerName; + + @QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String cronExpression; + + @QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = TimeZonePossibleValueSourceMetaDataProvider.NAME) + private String cronTimeZoneId; + + @QField(isRequired = true, maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = ScheduledJobType.NAME) + private String type; + + @QField(isRequired = true) + private Boolean isActive; + + @QAssociation(name = ScheduledJobParameter.TABLE_NAME) + private List jobParameters; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledJob() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledJob(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public ScheduledJob withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public ScheduledJob withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public ScheduledJob withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public ScheduledJob withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for description + *******************************************************************************/ + public String getDescription() + { + return (this.description); + } + + + + /******************************************************************************* + ** Setter for description + *******************************************************************************/ + public void setDescription(String description) + { + this.description = description; + } + + + + /******************************************************************************* + ** Fluent setter for description + *******************************************************************************/ + public ScheduledJob withDescription(String description) + { + this.description = description; + return (this); + } + + + + /******************************************************************************* + ** Getter for cronExpression + *******************************************************************************/ + public String getCronExpression() + { + return (this.cronExpression); + } + + + + /******************************************************************************* + ** Setter for cronExpression + *******************************************************************************/ + public void setCronExpression(String cronExpression) + { + this.cronExpression = cronExpression; + } + + + + /******************************************************************************* + ** Fluent setter for cronExpression + *******************************************************************************/ + public ScheduledJob withCronExpression(String cronExpression) + { + this.cronExpression = cronExpression; + return (this); + } + + + + /******************************************************************************* + ** Getter for cronTimeZoneId + *******************************************************************************/ + public String getCronTimeZoneId() + { + return (this.cronTimeZoneId); + } + + + + /******************************************************************************* + ** Setter for cronTimeZoneId + *******************************************************************************/ + public void setCronTimeZoneId(String cronTimeZoneId) + { + this.cronTimeZoneId = cronTimeZoneId; + } + + + + /******************************************************************************* + ** Fluent setter for cronTimeZoneId + *******************************************************************************/ + public ScheduledJob withCronTimeZoneId(String cronTimeZoneId) + { + this.cronTimeZoneId = cronTimeZoneId; + return (this); + } + + + + /******************************************************************************* + ** Getter for isActive + *******************************************************************************/ + public Boolean getIsActive() + { + return (this.isActive); + } + + + + /******************************************************************************* + ** Setter for isActive + *******************************************************************************/ + public void setIsActive(Boolean isActive) + { + this.isActive = isActive; + } + + + + /******************************************************************************* + ** Fluent setter for isActive + *******************************************************************************/ + public ScheduledJob withIsActive(Boolean isActive) + { + this.isActive = isActive; + return (this); + } + + + + /******************************************************************************* + ** Getter for schedulerName + *******************************************************************************/ + public String getSchedulerName() + { + return (this.schedulerName); + } + + + + /******************************************************************************* + ** Setter for schedulerName + *******************************************************************************/ + public void setSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + } + + + + /******************************************************************************* + ** Fluent setter for schedulerName + *******************************************************************************/ + public ScheduledJob withSchedulerName(String schedulerName) + { + this.schedulerName = schedulerName; + return (this); + } + + + + /******************************************************************************* + ** Getter for type + *******************************************************************************/ + public String getType() + { + return (this.type); + } + + + + /******************************************************************************* + ** Setter for type + *******************************************************************************/ + public void setType(String type) + { + this.type = type; + } + + + + /******************************************************************************* + ** Fluent setter for type + *******************************************************************************/ + public ScheduledJob withType(String type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** Getter for jobParameters + *******************************************************************************/ + public List getJobParameters() + { + return (this.jobParameters); + } + + + + /******************************************************************************* + ** Getter for jobParameters - but a map of just the key=value pairs. + *******************************************************************************/ + public Map getJobParametersMap() + { + if(CollectionUtils.nullSafeIsEmpty(this.jobParameters)) + { + return (new HashMap<>()); + } + + /////////////////////////////////////////////////////////////////////////////////////// + // wrap in mutable map, just to avoid any immutable or other bs from toMap's default // + /////////////////////////////////////////////////////////////////////////////////////// + return new MutableMap<>(jobParameters.stream().collect(Collectors.toMap(ScheduledJobParameter::getKey, ScheduledJobParameter::getValue))); + } + + + + /******************************************************************************* + ** Setter for jobParameters + *******************************************************************************/ + public void setJobParameters(List jobParameters) + { + this.jobParameters = jobParameters; + } + + + + /******************************************************************************* + ** Fluent setter for jobParameters + *******************************************************************************/ + public ScheduledJob withJobParameters(List jobParameters) + { + this.jobParameters = jobParameters; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobParameter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobParameter.java new file mode 100644 index 00000000..ccfa816e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobParameter.java @@ -0,0 +1,266 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobParameter extends QRecordEntity +{ + public static final String TABLE_NAME = "scheduledJobParameter"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(possibleValueSourceName = ScheduledJob.TABLE_NAME, isRequired = true) + private Integer scheduledJobId; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, isRequired = true) + private String key; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR) + private String value; + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledJobParameter() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ScheduledJobParameter(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public ScheduledJobParameter withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public ScheduledJobParameter withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public ScheduledJobParameter withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for scheduledJobId + *******************************************************************************/ + public Integer getScheduledJobId() + { + return (this.scheduledJobId); + } + + + + /******************************************************************************* + ** Setter for scheduledJobId + *******************************************************************************/ + public void setScheduledJobId(Integer scheduledJobId) + { + this.scheduledJobId = scheduledJobId; + } + + + + /******************************************************************************* + ** Fluent setter for scheduledJobId + *******************************************************************************/ + public ScheduledJobParameter withScheduledJobId(Integer scheduledJobId) + { + this.scheduledJobId = scheduledJobId; + return (this); + } + + + + /******************************************************************************* + ** Getter for key + *******************************************************************************/ + public String getKey() + { + return (this.key); + } + + + + /******************************************************************************* + ** Setter for key + *******************************************************************************/ + public void setKey(String key) + { + this.key = key; + } + + + + /******************************************************************************* + ** Fluent setter for key + *******************************************************************************/ + public ScheduledJobParameter withKey(String key) + { + this.key = key; + return (this); + } + + + + /******************************************************************************* + ** Getter for value + *******************************************************************************/ + public String getValue() + { + return (this.value); + } + + + + /******************************************************************************* + ** Setter for value + *******************************************************************************/ + public void setValue(String value) + { + this.value = value; + } + + + + /******************************************************************************* + ** Fluent setter for value + *******************************************************************************/ + public ScheduledJobParameter withValue(String value) + { + this.value = value; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java new file mode 100644 index 00000000..c8296e40 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java @@ -0,0 +1,123 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum ScheduledJobType implements PossibleValueEnum +{ + PROCESS, + QUEUE_PROCESSOR, + TABLE_AUTOMATIONS, + // todo - future - USER_REPORT + ; + + public static final String NAME = "scheduledJobType"; + + private final String label; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + ScheduledJobType() + { + this.label = QInstanceEnricher.nameToLabel(QInstanceEnricher.inferNameFromBackendName(name())); + } + + + + /******************************************************************************* + ** Get instance by id + ** + *******************************************************************************/ + public static ScheduledJobType getById(String id) + { + if(id == null) + { + return (null); + } + + for(ScheduledJobType value : ScheduledJobType.values()) + { + if(value.name().equals(id)) + { + return (value); + } + } + + return (null); + } + + + + /******************************************************************************* + ** Getter for id + ** + *******************************************************************************/ + public String getId() + { + return name(); + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getPossibleValueId() + { + return name(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return (label); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java new file mode 100644 index 00000000..92d0e22f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java @@ -0,0 +1,219 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers.ScheduledJobTableCustomizer; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobsMetaDataProvider +{ + private static final String JOB_PARAMETER_JOIN_NAME = QJoinMetaData.makeInferredJoinName(ScheduledJob.TABLE_NAME, ScheduledJobParameter.TABLE_NAME); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineAll(QInstance instance, String backendName, Consumer backendDetailEnricher) throws QException + { + defineStandardTables(instance, backendName, backendDetailEnricher); + instance.addPossibleValueSource(QPossibleValueSource.newForTable(ScheduledJob.TABLE_NAME)); + instance.addPossibleValueSource(QPossibleValueSource.newForEnum(ScheduledJobType.NAME, ScheduledJobType.values())); + instance.addPossibleValueSource(defineSchedulersPossibleValueSource()); + defineStandardJoins(instance); + defineStandardScriptsWidgets(instance); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineStandardScriptsWidgets(QInstance instance) + { + QJoinMetaData join = instance.getJoin(JOB_PARAMETER_JOIN_NAME); + instance.addWidget(ChildRecordListRenderer.widgetMetaDataBuilder(join) + .withCanAddChildRecord(true) + .withLabel("Parameters") + .getWidgetMetaData() + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineStandardJoins(QInstance instance) + { + instance.addJoin(new QJoinMetaData() + .withType(JoinType.ONE_TO_MANY) + .withLeftTable(ScheduledJob.TABLE_NAME) + .withRightTable(ScheduledJobParameter.TABLE_NAME) + .withJoinOn(new JoinOn("id", "scheduledJobId")) + .withOrderBy(new QFilterOrderBy("id")) + .withInferredName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineStandardTables(QInstance instance, String backendName, Consumer backendDetailEnricher) throws QException + { + for(QTableMetaData tableMetaData : defineStandardTables(backendName, backendDetailEnricher)) + { + instance.addTable(tableMetaData); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List defineStandardTables(String backendName, Consumer backendDetailEnricher) throws QException + { + List rs = new ArrayList<>(); + rs.add(enrich(backendDetailEnricher, defineScheduledJobTable(backendName))); + rs.add(enrich(backendDetailEnricher, defineScheduledJobParameterTable(backendName))); + return (rs); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData enrich(Consumer backendDetailEnricher, QTableMetaData table) + { + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineStandardTable(String backendName, String name, Class fieldsFromEntity) throws QException + { + return new QTableMetaData() + .withName(name) + .withBackendName(backendName) + .withPrimaryKeyField("id") + .withFieldsFromEntity(fieldsFromEntity); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineScheduledJobTable(String backendName) throws QException + { + QTableMetaData tableMetaData = defineStandardTable(backendName, ScheduledJob.TABLE_NAME, ScheduledJob.class) + .withRecordLabelFormat("%s") + .withRecordLabelFields("label") + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "description"))) + .withSection(new QFieldSection("schedule", new QIcon().withName("alarm"), Tier.T2, List.of("cronExpression", "cronTimeZoneId"))) + .withSection(new QFieldSection("settings", new QIcon().withName("tune"), Tier.T2, List.of("type", "isActive", "schedulerName"))) + .withSection(new QFieldSection("parameters", new QIcon().withName("list"), Tier.T2).withWidgetName(JOB_PARAMETER_JOIN_NAME)) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + QCodeReference customizerReference = new QCodeReference(ScheduledJobTableCustomizer.class); + tableMetaData.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_INSERT_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_DELETE_RECORD, customizerReference); + + tableMetaData.withAssociation(new Association() + .withAssociatedTableName(ScheduledJobParameter.TABLE_NAME) + .withJoinName(JOB_PARAMETER_JOIN_NAME) + .withName(ScheduledJobParameter.TABLE_NAME)); + + return (tableMetaData); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QTableMetaData defineScheduledJobParameterTable(String backendName) throws QException + { + QTableMetaData tableMetaData = defineStandardTable(backendName, ScheduledJobParameter.TABLE_NAME, ScheduledJobParameter.class) + .withRecordLabelFormat("%s - %s") + .withRecordLabelFields("scheduledJobId", "key") + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "scheduledJobId", "key"))) + .withSection(new QFieldSection("value", new QIcon().withName("dataset"), Tier.T2, List.of("value"))) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + return (tableMetaData); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QPossibleValueSource defineSchedulersPossibleValueSource() + { + return (new QPossibleValueSource() + .withName(SchedulersPossibleValueSource.NAME) + .withType(QPossibleValueSourceType.CUSTOM) + .withCustomCodeReference(new QCodeReference(SchedulersPossibleValueSource.class))); + + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java new file mode 100644 index 00000000..a2363267 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java @@ -0,0 +1,87 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SchedulersPossibleValueSource implements QCustomPossibleValueProvider +{ + public static final String NAME = "schedulers"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QPossibleValue getPossibleValue(Serializable idValue) + { + QSchedulerMetaData scheduler = QContext.getQInstance().getScheduler(String.valueOf(idValue)); + if(scheduler != null) + { + return schedulerToPossibleValue(scheduler); + } + + return null; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List> search(SearchPossibleValueSourceInput input) throws QException + { + List> rs = new ArrayList<>(); + for(QSchedulerMetaData scheduler : CollectionUtils.nonNullMap(QContext.getQInstance().getSchedulers()).values()) + { + rs.add(schedulerToPossibleValue(scheduler)); + } + return rs; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QPossibleValue schedulerToPossibleValue(QSchedulerMetaData scheduler) + { + return new QPossibleValue<>(scheduler.getName(), scheduler.getName()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java new file mode 100644 index 00000000..0f4a4558 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java @@ -0,0 +1,278 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers; + + +import java.io.Serializable; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobTableCustomizer implements TableCustomizerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + validateConditionalFields(records); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + scheduleJobsForRecordList(records); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + validateConditionalFields(records); + + if(isPreview || oldRecordList.isEmpty()) + { + return (records); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // refresh the old-records w/ versions that have associations - so we can use those in the post-update to property unschedule things // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Map freshOldRecordsWithAssociationsMap = CollectionUtils.recordsToMap(freshlyQueryForRecordsWithAssociations(oldRecordList.get()), "id"); + ListIterator iterator = oldRecordList.get().listIterator(); + while(iterator.hasNext()) + { + QRecord record = iterator.next(); + QRecord freshRecord = freshOldRecordsWithAssociationsMap.get(record.getValue("id")); + if(freshRecord != null) + { + iterator.set(freshRecord); + } + } + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void validateConditionalFields(List records) + { + for(QRecord record : records) + { + if(StringUtils.hasContent(record.getValueString("cronExpression"))) + { + if(!StringUtils.hasContent(record.getValueString("cronTimeZoneId"))) + { + record.addError(new BadInputStatusMessage("If a Cron Expression is given, then a Cron Time Zone Id is required.")); + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + if(oldRecordList.isPresent()) + { + Set idsWithErrors = getRecordIdsWithErrors(records); + unscheduleJobsForRecordList(oldRecordList.get(), idsWithErrors); + } + + scheduleJobsForRecordList(records); + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Set getRecordIdsWithErrors(List records) + { + return records.stream() + .filter(r -> !recordHasErrors().test(r)) + .map(r -> r.getValueInteger("id")) + .collect(Collectors.toSet()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postDelete(DeleteInput deleteInput, List records) throws QException + { + Set idsWithErrors = getRecordIdsWithErrors(records); + unscheduleJobsForRecordList(records, idsWithErrors); + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void scheduleJobsForRecordList(List records) + { + List recordsWithoutErrors = records.stream().filter(recordHasErrors()).toList(); + if(CollectionUtils.nullSafeIsEmpty(recordsWithoutErrors)) + { + return; + } + + try + { + List freshRecordListWithAssociations = freshlyQueryForRecordsWithAssociations(recordsWithoutErrors); + + QScheduleManager scheduleManager = QScheduleManager.getInstance(); + for(QRecord record : freshRecordListWithAssociations) + { + try + { + scheduleManager.setupScheduledJob(new ScheduledJob(record)); + } + catch(Exception e) + { + LOG.info("Caught exception while scheduling a job in post-action", e, logPair("id", record.getValue("id"))); + } + } + } + catch(Exception e) + { + LOG.warn("Error scheduling jobs in post-action", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Predicate recordHasErrors() + { + return r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static List freshlyQueryForRecordsWithAssociations(List records) throws QException + { + List idList = records.stream().map(r -> r.getValueInteger("id")).toList(); + + return new QueryAction().execute(new QueryInput(ScheduledJob.TABLE_NAME) + .withIncludeAssociations(true) + .withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, idList)))) + .getRecords(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void unscheduleJobsForRecordList(List oldRecords, Set exceptIdsWithErrors) + { + try + { + QScheduleManager scheduleManager = QScheduleManager.getInstance(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for un-schedule - use the old records as they are - don't re-query them (they may not exist anymore!) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : oldRecords) + { + try + { + ScheduledJob scheduledJob = new ScheduledJob(record); + + if(exceptIdsWithErrors.contains(scheduledJob.getId())) + { + LOG.info("Will not unschedule the job for a record that had an error", logPair("id", scheduledJob.getId())); + continue; + } + + scheduleManager.unscheduleScheduledJob(scheduledJob); + } + catch(Exception e) + { + LOG.info("Caught exception while scheduling a job in post-action", e, logPair("id", record.getValue("id"))); + } + } + } + catch(Exception e) + { + LOG.warn("Error scheduling jobs in post-action", e); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index e3edee1e..6f6b0435 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -28,12 +28,15 @@ import java.util.List; import java.util.Map; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.CapturedContext; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.logging.LogPair; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -44,11 +47,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaDa import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; import org.apache.commons.lang.NotImplementedException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -112,21 +117,25 @@ public class QScheduleManager *******************************************************************************/ public void start() throws QException { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // initialize the scheduler(s) we're configured to use // + // do this, even if we won't start them - so, for example, a web server can still be aware of schedules in the application // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QSchedulerMetaData schedulerMetaData : CollectionUtils.nonNullMap(qInstance.getSchedulers()).values()) + { + QSchedulerInterface scheduler = schedulerMetaData.initSchedulerInstance(qInstance, systemUserSessionSupplier); + schedulers.put(schedulerMetaData.getName(), scheduler); + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now, exist w/o setting up schedules and not starting schedules, if schedule manager isn't enabled here // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) { LOG.info("Not starting ScheduleManager per settings."); return; } - ///////////////////////////////////////////////////////// - // initialize the scheduler(s) we're configured to use // - ///////////////////////////////////////////////////////// - for(QSchedulerMetaData schedulerMetaData : qInstance.getSchedulers().values()) - { - QSchedulerInterface scheduler = schedulerMetaData.initSchedulerInstance(qInstance, systemUserSessionSupplier); - schedulers.put(schedulerMetaData.getName(), scheduler); - } - ///////////////////////////////////////////////////////////////////////////////////////////////// // ensure that everything which should be scheduled is scheduled, in the appropriate scheduler // ///////////////////////////////////////////////////////////////////////////////////////////////// @@ -165,6 +174,26 @@ public class QScheduleManager *******************************************************************************/ private void setupSchedules() { + ///////////////////////////////////////////// + // read dynamic schedules // + // e.g., user-scheduled processes, reports // + ///////////////////////////////////////////// + List scheduledJobList = null; + try + { + if(QContext.getQInstance().getTables().containsKey(ScheduledJob.TABLE_NAME)) + { + scheduledJobList = new QueryAction() + .execute(new QueryInput(ScheduledJob.TABLE_NAME) + .withIncludeAssociations(true)) + .getRecordEntities(ScheduledJob.class); + } + } + catch(Exception e) + { + throw (new QRuntimeException("Failed to query for scheduled jobs - will not set up scheduler!", e)); + } + ///////////////////////////////////////////////////////// // let the schedulers know we're starting this process // ///////////////////////////////////////////////////////// @@ -193,46 +222,25 @@ public class QScheduleManager { if(process.getSchedule() != null) { - QScheduleMetaData scheduleMetaData = process.getSchedule(); - if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) - { - /////////////////////////////////////////////// - // if no variants, or variant is serial mode // - /////////////////////////////////////////////// - setupProcess(process, null); - } - else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) - { - ///////////////////////////////////////////////////////////////////////////////////////////////////// - // if this a "parallel", which for example means we want to have a thread for each backend variant // - // running at the same time, get the variant records and schedule each separately // - ///////////////////////////////////////////////////////////////////////////////////////////////////// - QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); - for(QRecord qRecord : CollectionUtils.nonNullList(SchedulerUtils.getBackendVariantFilteredRecords(process))) - { - try - { - setupProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); - } - catch(Exception e) - { - LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); - } - } - } - else - { - LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided."); - } + setupProcess(process); } } - ///////////////////////////////////////////////////////////// - // todo - read dynamic schedules and schedule those things // - // e.g., user-scheduled processes, reports // - ///////////////////////////////////////////////////////////// - // ScheduledJob scheduledJob = new ScheduledJob(); - // setupScheduledJob(scheduledJob); + ///////////////////////////////////////////////////////////////////////////////////////////// + // todo- before, or after meta-datas? // + // like quartz, it'd just re-schedule if a dupe - but, should we do our own dupe checking? // + ///////////////////////////////////////////////////////////////////////////////////////////// + for(ScheduledJob scheduledJob : CollectionUtils.nonNullList(scheduledJobList)) + { + try + { + setupScheduledJob(scheduledJob); + } + catch(Exception e) + { + LOG.info("Caught exception while scheduling a job", e, logPair("id", scheduledJob.getId())); + } + } ////////////////////////////////////////////////////////// // let the schedulers know we're done with this process // @@ -242,6 +250,122 @@ public class QScheduleManager + /******************************************************************************* + ** + *******************************************************************************/ + public void setupScheduledJob(ScheduledJob scheduledJob) + { + /////////////////////////////////////////////////////////////////////////////////////////// + // non-active jobs should be deleted from the scheduler. they get re-added // + // if they get re-activated. but we don't want to rely on (e.g., for quartz) the paused // + // state to be drive by is-active. else, devops-pause & unpause ops would clobber // + // scheduled-job record facts // + /////////////////////////////////////////////////////////////////////////////////////////// + if(!scheduledJob.getIsActive()) + { + unscheduleScheduledJob(scheduledJob); + return; + } + + QSchedulerInterface scheduler = getScheduler(scheduledJob.getSchedulerName()); + + QScheduleMetaData scheduleMetaData = new QScheduleMetaData(); + scheduleMetaData.setCronExpression(scheduledJob.getCronExpression()); + scheduleMetaData.setCronTimeZoneId(scheduledJob.getCronTimeZoneId()); + + switch(ScheduledJobType.getById(scheduledJob.getType())) + { + case PROCESS -> + { + Map paramMap = scheduledJob.getJobParametersMap(); + String processName = paramMap.get("processName"); + QProcessMetaData process = qInstance.getProcess(processName); + + // todo - variants... serial vs parallel? + scheduler.setupProcess(process, null, scheduleMetaData, true); + } + case QUEUE_PROCESSOR -> + { + throw new NotImplementedException("ScheduledJob queue processors are not yet implemented..."); + } + case TABLE_AUTOMATIONS -> + { + throw new NotImplementedException("ScheduledJob table automations are not yet implemented..."); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void unscheduleScheduledJob(ScheduledJob scheduledJob) + { + QSchedulerInterface scheduler = getScheduler(scheduledJob.getSchedulerName()); + + switch(ScheduledJobType.getById(scheduledJob.getType())) + { + case PROCESS -> + { + Map paramMap = scheduledJob.getJobParametersMap(); + String processName = paramMap.get("processName"); + QProcessMetaData process = qInstance.getProcess(processName); + scheduler.unscheduleProcess(process); + } + case QUEUE_PROCESSOR -> + { + throw new NotImplementedException("ScheduledJob queue processors are not yet implemented..."); + } + case TABLE_AUTOMATIONS -> + { + throw new NotImplementedException("ScheduledJob table automations are not yet implemented..."); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setupProcess(QProcessMetaData process) + { + QScheduleMetaData scheduleMetaData = process.getSchedule(); + if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) + { + /////////////////////////////////////////////// + // if no variants, or variant is serial mode // + /////////////////////////////////////////////// + setupProcess(process, null); + } + else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // if this a "parallel", which for example means we want to have a thread for each backend variant // + // running at the same time, get the variant records and schedule each separately // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); + for(QRecord qRecord : CollectionUtils.nonNullList(SchedulerUtils.getBackendVariantFilteredRecords(process))) + { + try + { + setupProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); + } + catch(Exception e) + { + LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord)); + } + } + } + else + { + LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided."); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java index 3e61c77f..1e18d863 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java @@ -52,6 +52,11 @@ public interface QSchedulerInterface *******************************************************************************/ void setupTableAutomation(QAutomationProviderMetaData automationProvider, PollingAutomationPerTableRunner.TableActionsInterface tableActions, QScheduleMetaData schedule, boolean allowedToStart); + /******************************************************************************* + ** + *******************************************************************************/ + void unscheduleProcess(QProcessMetaData process); + /******************************************************************************* ** *******************************************************************************/ @@ -90,5 +95,4 @@ public interface QSchedulerInterface { } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java index 1e3189c7..44220452 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java @@ -83,6 +83,11 @@ public class QuartzScheduler implements QSchedulerInterface private Scheduler scheduler; + private final static String GROUP_NAME_PROCESSES = "processes"; + private final static String GROUP_NAME_SQS_QUEUES = "sqsQueues"; + private final static String GROUP_NAME_TABLE_AUTOMATIONS = "tableAutomations"; + + ///////////////////////////////////////////////////////////////////////////////////////// // create memoization objects for some quartz-query functions, that we'll only want to // // use during our setup routine, when we'd query it many times over and over again. // @@ -95,12 +100,18 @@ public class QuartzScheduler implements QSchedulerInterface private Memoization> jobKeyNamesMemoization = new Memoization>() .withTimeout(Duration.of(0, ChronoUnit.SECONDS)); + private Memoization> queryQuartzMemoization = new Memoization>() + .withTimeout(Duration.of(0, ChronoUnit.SECONDS)); + + private List> allMemoizations = List.of(jobGroupNamesMemoization, jobKeyNamesMemoization, queryQuartzMemoization); + /////////////////////////////////////////////////////////////////////////////// // vars used during the setup routine, to figure out what jobs need deleted. // /////////////////////////////////////////////////////////////////////////////// - private boolean insideSetup = false; + private boolean insideSetup = false; private List scheduledJobsAtStartOfSetup = new ArrayList<>(); - private List scheduledJobsAtEndOfSetup = new ArrayList<>(); + private List scheduledJobsAtEndOfSetup = new ArrayList<>(); + /******************************************************************************* @@ -227,7 +238,7 @@ public class QuartzScheduler implements QSchedulerInterface jobData.put("backendVariantData", backendVariantData); } - scheduleJob(process.getName(), "processes", QuartzRunProcessJob.class, jobData, schedule, allowedToStart); + scheduleJob(process.getName(), GROUP_NAME_PROCESSES, QuartzRunProcessJob.class, jobData, schedule, allowedToStart); } @@ -239,8 +250,7 @@ public class QuartzScheduler implements QSchedulerInterface public void startOfSetupSchedules() { this.insideSetup = true; - this.jobGroupNamesMemoization.setTimeout(Duration.ofSeconds(5)); - this.jobKeyNamesMemoization.setTimeout(Duration.ofSeconds(5)); + this.allMemoizations.forEach(m -> m.setTimeout(Duration.ofSeconds(5))); try { @@ -253,6 +263,7 @@ public class QuartzScheduler implements QSchedulerInterface } + /******************************************************************************* ** *******************************************************************************/ @@ -260,8 +271,7 @@ public class QuartzScheduler implements QSchedulerInterface public void endOfSetupSchedules() { this.insideSetup = false; - this.jobGroupNamesMemoization.setTimeout(Duration.ofSeconds(0)); - this.jobKeyNamesMemoization.setTimeout(Duration.ofSeconds(0)); + this.allMemoizations.forEach(m -> m.setTimeout(Duration.ofSeconds(0))); if(this.scheduledJobsAtStartOfSetup == null) { @@ -271,7 +281,7 @@ public class QuartzScheduler implements QSchedulerInterface try { Set startJobKeys = this.scheduledJobsAtStartOfSetup.stream().map(w -> w.jobDetail().getKey()).collect(Collectors.toSet()); - Set endJobKeys = scheduledJobsAtEndOfSetup.stream().map(w -> w.jobDetail().getKey()).collect(Collectors.toSet()); + Set endJobKeys = scheduledJobsAtEndOfSetup.stream().map(w -> w.jobDetail().getKey()).collect(Collectors.toSet()); ///////////////////////////////////////////////////////////////////////////////////////////////////// // remove all 'end' keys from the set of start keys. any left-over start-keys need to be deleted. // @@ -343,6 +353,14 @@ public class QuartzScheduler implements QSchedulerInterface { startAt.setTime(startAt.getTime() + scheduleMetaData.getInitialDelaySeconds() * 1000); } + else + { + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // by default, put a 3-second delay on everything we schedule // + // this gives us a chance to re-pause if the job was previously paused, but then we re-schedule it. // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + startAt.setTime(startAt.getTime() + 3000); + } /////////////////////////////////////// // Define a Trigger for the schedule // @@ -359,18 +377,6 @@ public class QuartzScheduler implements QSchedulerInterface /////////////////////////////////////// addOrReplaceJobAndTrigger(jobKey, jobDetail, trigger); - ////////////////////////////////////////////////////////// - // either pause or resume, based on if allowed to start // - ////////////////////////////////////////////////////////// - if(!allowedToStart) - { - pauseJob(jobKey.getName(), jobKey.getGroup()); - } - else - { - resumeJob(jobKey.getName(), jobKey.getGroup()); - } - /////////////////////////////////////////////////////////////////////////// // if we're inside the setup event (e.g., initial startup), then capture // // this job as one that is currently active and should be kept. // @@ -404,10 +410,11 @@ public class QuartzScheduler implements QSchedulerInterface jobData.put("queueProviderName", queueProvider.getName()); jobData.put("queueName", queue.getName()); - scheduleJob(queue.getName(), "sqsQueue", QuartzSqsPollerJob.class, jobData, schedule, allowedToStart); + scheduleJob(queue.getName(), GROUP_NAME_SQS_QUEUES, QuartzSqsPollerJob.class, jobData, schedule, allowedToStart); } + /******************************************************************************* ** *******************************************************************************/ @@ -422,10 +429,22 @@ public class QuartzScheduler implements QSchedulerInterface jobData.put("tableName", tableActions.tableName()); jobData.put("automationStatus", tableActions.status().toString()); - scheduleJob(tableActions.tableName() + "." + tableActions.status(), "tableAutomations", QuartzTableAutomationsJob.class, jobData, schedule, allowedToStart); + scheduleJob(tableActions.tableName() + "." + tableActions.status(), GROUP_NAME_TABLE_AUTOMATIONS, QuartzTableAutomationsJob.class, jobData, schedule, allowedToStart); } + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unscheduleProcess(QProcessMetaData process) + { + deleteJob(new JobKey(process.getName(), GROUP_NAME_PROCESSES)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -434,17 +453,42 @@ public class QuartzScheduler implements QSchedulerInterface boolean isJobAlreadyScheduled = isJobAlreadyScheduled(jobKey); if(isJobAlreadyScheduled) { - this.scheduler.addJob(jobDetail, true); + boolean wasPaused = wasExistingJobPaused(jobKey); + + this.scheduler.addJob(jobDetail, true); // note, true flag here replaces if already present. this.scheduler.rescheduleJob(trigger.getKey(), trigger); - LOG.info("Re-scheduled job: " + jobKey); + LOG.info("Re-scheduled job", logPair("jobKey", jobKey)); + if(wasPaused) + { + LOG.info("Re-pausing job", logPair("jobKey", jobKey)); + pauseJob(jobKey.getName(), jobKey.getGroup()); + } } else { this.scheduler.scheduleJob(jobDetail, trigger); - LOG.info("Scheduled new job: " + jobKey); + LOG.info("Scheduled new job", logPair("jobKey", jobKey)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean wasExistingJobPaused(JobKey jobKey) throws SchedulerException + { + List quartzJobAndTriggerWrappers = queryQuartz(); + Optional existingWrapper = quartzJobAndTriggerWrappers.stream().filter(w -> w.jobDetail().getKey().equals(jobKey)).findFirst(); + if(existingWrapper.isPresent()) + { + if(Trigger.TriggerState.PAUSED.equals(existingWrapper.get().triggerState())) + { + return (true); + } } - // todo - think about... clear memoization - but - when this is used in bulk, that's when we want the memo! + return(false); } @@ -492,12 +536,15 @@ public class QuartzScheduler implements QSchedulerInterface ///////////////////////////////////////////////////////////////////////////////////////////// if(isJobAlreadyScheduled(jobKey)) { - return scheduler.deleteJob(jobKey); + boolean result = scheduler.deleteJob(jobKey); + LOG.info("Attempted to delete quartz job", logPair("jobKey", jobKey), logPair("deleteJobResult", result)); + return (result); } ///////////////////////////////////////// // return true to indicate, we're good // ///////////////////////////////////////// + LOG.info("Request to delete quartz job, but it is not already scheduled.", logPair("jobKey", jobKey)); return (true); } catch(Exception e) @@ -576,25 +623,27 @@ public class QuartzScheduler implements QSchedulerInterface *******************************************************************************/ List queryQuartz() throws SchedulerException { - List rs = new ArrayList<>(); - List jobGroupNames = scheduler.getJobGroupNames(); - - for(String group : jobGroupNames) + return queryQuartzMemoization.getResultThrowing(AnyKey.getInstance(), (x) -> { - Set jobKeys = scheduler.getJobKeys(GroupMatcher.groupEquals(group)); - for(JobKey jobKey : jobKeys) + List rs = new ArrayList<>(); + + for(String group : scheduler.getJobGroupNames()) { - JobDetail jobDetail = scheduler.getJobDetail(jobKey); - List triggersOfJob = scheduler.getTriggersOfJob(jobKey); - for(Trigger trigger : triggersOfJob) + Set jobKeys = scheduler.getJobKeys(GroupMatcher.groupEquals(group)); + for(JobKey jobKey : jobKeys) { - Trigger.TriggerState triggerState = scheduler.getTriggerState(trigger.getKey()); - rs.add(new QuartzJobAndTriggerWrapper(jobDetail, trigger, triggerState)); + JobDetail jobDetail = scheduler.getJobDetail(jobKey); + List triggersOfJob = scheduler.getTriggersOfJob(jobKey); + for(Trigger trigger : triggersOfJob) + { + Trigger.TriggerState triggerState = scheduler.getTriggerState(trigger.getKey()); + rs.add(new QuartzJobAndTriggerWrapper(jobDetail, trigger, triggerState)); + } } } - } - return (rs); + return (rs); + }).orElse(null); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDetailsPostQueryCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDataPostQueryCustomizer.java similarity index 86% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDetailsPostQueryCustomizer.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDataPostQueryCustomizer.java index 3c08d4ba..372a51e3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDetailsPostQueryCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDataPostQueryCustomizer.java @@ -33,9 +33,9 @@ import org.apache.commons.lang3.SerializationUtils; /******************************************************************************* ** *******************************************************************************/ -public class QuartzJobDetailsPostQueryCustomizer extends AbstractPostQueryCustomizer +public class QuartzJobDataPostQueryCustomizer extends AbstractPostQueryCustomizer { - private static final QLogger LOG = QLogger.getLogger(QuartzJobDetailsPostQueryCustomizer.class); + private static final QLogger LOG = QLogger.getLogger(QuartzJobDataPostQueryCustomizer.class); @@ -55,9 +55,12 @@ public class QuartzJobDetailsPostQueryCustomizer extends AbstractPostQueryCustom // this field has a blob of essentially a serialized map - so, deserialize that, then convert to JSON // //////////////////////////////////////////////////////////////////////////////////////////////////////// byte[] value = record.getValueByteArray("jobData"); - Object deserialize = SerializationUtils.deserialize(value); - String json = JsonUtils.toJson(deserialize); - record.setValue("jobData", json); + if(value.length > 0) + { + Object deserialize = SerializationUtils.deserialize(value); + String json = JsonUtils.toJson(deserialize); + record.setValue("jobData", json); + } } catch(Exception e) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java index af062d0b..ac7ab640 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java @@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaDa import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; +import org.apache.commons.lang.NotImplementedException; /******************************************************************************* @@ -157,6 +158,17 @@ public class SimpleScheduler implements QSchedulerInterface + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unscheduleProcess(QProcessMetaData process) + { + throw (new NotImplementedException("Unscheduling is not implemented in SimpleScheduler...")); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java index 743db22d..3c73cbd0 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java @@ -177,7 +177,7 @@ class QuartzJobsProcessTest extends BaseTest List quartzJobAndTriggerWrappers = QuartzTestUtils.queryQuartz(); new InsertAction().execute(new InsertInput("quartzTriggers").withRecord(new QRecord() .withValue("jobName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()) - .withValue("groupName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getGroup()) + .withValue("jobGroup", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getGroup()) )); input = new RunProcessInput(); @@ -224,7 +224,7 @@ class QuartzJobsProcessTest extends BaseTest List quartzJobAndTriggerWrappers = QuartzTestUtils.queryQuartz(); new InsertAction().execute(new InsertInput("quartzTriggers").withRecord(new QRecord() .withValue("jobName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()) - .withValue("groupName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getGroup()) + .withValue("jobGroup", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getGroup()) )); RunProcessInput input = new RunProcessInput(); From aa69f0e7d7119bfb19159ffe1ee0838cb67113c9 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 14 Mar 2024 11:27:33 -0500 Subject: [PATCH 43/72] Checkstyle --- .../qqq/backend/core/scheduler/QScheduleManager.java | 2 ++ .../qqq/backend/core/scheduler/quartz/QuartzScheduler.java | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index 6f6b0435..da9ca9b8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -292,6 +292,7 @@ public class QScheduleManager { throw new NotImplementedException("ScheduledJob table automations are not yet implemented..."); } + default -> throw new IllegalStateException("Unexpected value: " + ScheduledJobType.getById(scheduledJob.getType())); } } @@ -321,6 +322,7 @@ public class QScheduleManager { throw new NotImplementedException("ScheduledJob table automations are not yet implemented..."); } + default -> throw new IllegalStateException("Unexpected value: " + ScheduledJobType.getById(scheduledJob.getType())); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java index 44220452..6f682875 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java @@ -83,9 +83,9 @@ public class QuartzScheduler implements QSchedulerInterface private Scheduler scheduler; - private final static String GROUP_NAME_PROCESSES = "processes"; - private final static String GROUP_NAME_SQS_QUEUES = "sqsQueues"; - private final static String GROUP_NAME_TABLE_AUTOMATIONS = "tableAutomations"; + private static final String GROUP_NAME_PROCESSES = "processes"; + private static final String GROUP_NAME_SQS_QUEUES = "sqsQueues"; + private static final String GROUP_NAME_TABLE_AUTOMATIONS = "tableAutomations"; ///////////////////////////////////////////////////////////////////////////////////////// From c093c680c094152b6789d5af1a347b46358bfd33 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 14 Mar 2024 11:33:28 -0500 Subject: [PATCH 44/72] Work-around default delay when scheduling, for test --- .../core/scheduler/quartz/QuartzTestUtils.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java index 44ba5437..3e952b50 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java @@ -72,19 +72,25 @@ public class QuartzTestUtils .withProperties(getQuartzProperties()) .withName(QUARTZ_SCHEDULER_NAME)); - //////////////////////////////////////////////////////////////////////////////// - // set the queue providers & automation providers to use the quartz scheduler // - //////////////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////////// + // set the queue providers & automation providers to use the quartz scheduler // + // also, set their initial delay to avoid default delay done by our scheduler // + // (that gives us a chance to re-pause if re-scheduling a previously paused job) // + /////////////////////////////////////////////////////////////////////////////////// qInstance.getTables().values().forEach(t -> { if(t.getAutomationDetails() != null) { - t.getAutomationDetails().getSchedule().setSchedulerName(QUARTZ_SCHEDULER_NAME); + t.getAutomationDetails().getSchedule() + .withSchedulerName(QUARTZ_SCHEDULER_NAME) + .withInitialDelayMillis(1); } }); qInstance.getQueues().values() - .forEach(q -> q.getSchedule().setSchedulerName(QUARTZ_SCHEDULER_NAME)); + .forEach(q -> q.getSchedule() + .withSchedulerName(QUARTZ_SCHEDULER_NAME) + .withInitialDelayMillis(1)); } From 1bffd4d46e08325f3be28436fd3417cdbcca6d87 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Mar 2024 10:53:05 -0500 Subject: [PATCH 45/72] CE-936 - Move process variant info from schedule to process meta data --- .../actions/processes/RunProcessAction.java | 4 +- .../core/instances/QInstanceValidator.java | 14 ++- .../AbstractProcessMetaDataBuilder.java | 22 +++++ .../metadata/processes/QProcessMetaData.java | 65 ++++++++++++ .../processes/VariantRunStrategy.java | 32 ++++++ .../scheduleing/QScheduleMetaData.java | 98 ++++++------------- .../StreamedETLWithFrontendProcess.java | 24 +++++ .../tablesync/TableSyncProcess.java | 24 +++++ 8 files changed, 209 insertions(+), 74 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/VariantRunStrategy.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index c9350e68..8774d9ae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -497,10 +497,10 @@ public class RunProcessAction ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if backend specifies that it uses variants, look for that data in the session and append to our basepull key // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(process.getSchedule() != null && process.getSchedule().getVariantBackend() != null) + if(process.getSchedule() != null && process.getVariantBackend() != null) { QSession session = QContext.getQSession(); - QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getSchedule().getVariantBackend()); + QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend()); if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue())) { LOG.info("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index bc4dc01d..43e1814e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -1417,12 +1417,16 @@ public class QInstanceValidator { QScheduleMetaData schedule = process.getSchedule(); validateScheduleMetaData(schedule, qInstance, "Process " + processName + ", schedule: "); + } - if(schedule.getVariantBackend() != null) - { - assertCondition(qInstance.getBackend(schedule.getVariantBackend()) != null, "A variant backend was not found for " + schedule.getVariantBackend()); - assertCondition(schedule.getVariantRunStrategy() != null, "A variant run strategy was not set for " + schedule.getVariantBackend() + " on schedule in process " + processName); - } + if(process.getVariantBackend() != null) + { + assertCondition(qInstance.getBackend(process.getVariantBackend()) != null, "Process " + processName + ", a variant backend was not found named " + process.getVariantBackend()); + assertCondition(process.getVariantRunStrategy() != null, "A variant run strategy was not set for process " + processName + " (which does specify a variant backend)"); + } + else + { + assertCondition(process.getVariantRunStrategy() == null, "A variant run strategy was set for process " + processName + " (which isn't allowed, since it does not specify a variant backend)"); } for(QSupplementalProcessMetaData supplementalProcessMetaData : CollectionUtils.nonNullMap(process.getSupplementalMetaData()).values()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java index 5f60e4b0..e14df8bc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java @@ -155,4 +155,26 @@ public class AbstractProcessMetaDataBuilder return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public AbstractProcessMetaDataBuilder withVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + processMetaData.setVariantRunStrategy(variantRunStrategy); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public AbstractProcessMetaDataBuilder withVariantBackend(String variantBackend) + { + processMetaData.setVariantBackend(variantBackend); + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index 8a1a5eae..fae291e8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -64,6 +64,9 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi private QScheduleMetaData schedule; + private VariantRunStrategy variantRunStrategy; + private String variantBackend; + private Map supplementalMetaData; @@ -671,4 +674,66 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi return (this); } + + /******************************************************************************* + ** Getter for variantRunStrategy + *******************************************************************************/ + public VariantRunStrategy getVariantRunStrategy() + { + return (this.variantRunStrategy); + } + + + + /******************************************************************************* + ** Setter for variantRunStrategy + *******************************************************************************/ + public void setVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + this.variantRunStrategy = variantRunStrategy; + } + + + + /******************************************************************************* + ** Fluent setter for variantRunStrategy + *******************************************************************************/ + public QProcessMetaData withVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + this.variantRunStrategy = variantRunStrategy; + return (this); + } + + + + /******************************************************************************* + ** Getter for variantBackend + *******************************************************************************/ + public String getVariantBackend() + { + return (this.variantBackend); + } + + + + /******************************************************************************* + ** Setter for variantBackend + *******************************************************************************/ + public void setVariantBackend(String variantBackend) + { + this.variantBackend = variantBackend; + } + + + + /******************************************************************************* + ** Fluent setter for variantBackend + *******************************************************************************/ + public QProcessMetaData withVariantBackend(String variantBackend) + { + this.variantBackend = variantBackend; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/VariantRunStrategy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/VariantRunStrategy.java new file mode 100644 index 00000000..888cc3ae --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/VariantRunStrategy.java @@ -0,0 +1,32 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.processes; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum VariantRunStrategy +{ + PARALLEL, + SERIAL +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java index 8dd19d96..d474a013 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java @@ -37,11 +37,8 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class QScheduleMetaData { - public enum RunStrategy - {PARALLEL, SERIAL} - - private String schedulerName; + private String description; private Integer repeatSeconds; private Integer repeatMillis; @@ -51,8 +48,6 @@ public class QScheduleMetaData private String cronExpression; private String cronTimeZoneId; - private RunStrategy variantRunStrategy; - private String variantBackend; /******************************************************************************* @@ -201,67 +196,6 @@ public class QScheduleMetaData - /******************************************************************************* - ** Getter for variantBackend - *******************************************************************************/ - public String getVariantBackend() - { - return (this.variantBackend); - } - - - - /******************************************************************************* - ** Setter for variantBackend - *******************************************************************************/ - public void setVariantBackend(String variantBackend) - { - this.variantBackend = variantBackend; - } - - - - /******************************************************************************* - ** Fluent setter for variantBackend - *******************************************************************************/ - public QScheduleMetaData withBackendVariant(String backendVariant) - { - this.variantBackend = backendVariant; - return (this); - } - - - - /******************************************************************************* - ** Getter for variantRunStrategy - *******************************************************************************/ - public RunStrategy getVariantRunStrategy() - { - return (this.variantRunStrategy); - } - - - - /******************************************************************************* - ** Setter for variantRunStrategy - *******************************************************************************/ - public void setVariantRunStrategy(RunStrategy variantRunStrategy) - { - this.variantRunStrategy = variantRunStrategy; - } - - - - /******************************************************************************* - ** Fluent setter for variantRunStrategy - *******************************************************************************/ - public QScheduleMetaData withVariantRunStrategy(RunStrategy variantRunStrategy) - { - this.variantRunStrategy = variantRunStrategy; - return (this); - } - - /******************************************************************************* ** Getter for cronExpression *******************************************************************************/ @@ -354,4 +288,34 @@ public class QScheduleMetaData } + /******************************************************************************* + ** Getter for description + *******************************************************************************/ + public String getDescription() + { + return (this.description); + } + + + + /******************************************************************************* + ** Setter for description + *******************************************************************************/ + public void setDescription(String description) + { + this.description = description; + } + + + + /******************************************************************************* + ** Fluent setter for description + *******************************************************************************/ + public QScheduleMetaData withDescription(String description) + { + this.description = description; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 9c279017..7eb3dd48 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.VariantRunStrategy; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; @@ -488,5 +489,28 @@ public class StreamedETLWithFrontendProcess return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Builder withVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + processMetaData.setVariantRunStrategy(variantRunStrategy); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Builder withVariantBackend(String variantBackend) + { + processMetaData.setVariantBackend(variantBackend); + return (this); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java index f17b31e0..5a2543b9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/TableSyncProcess.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.VariantRunStrategy; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.ExtractViaBasepullQueryStep; @@ -248,5 +249,28 @@ public class TableSyncProcess super.withExtractStepClass(extractStepClass); return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Builder withVariantRunStrategy(VariantRunStrategy variantRunStrategy) + { + processMetaData.setVariantRunStrategy(variantRunStrategy); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Builder withVariantBackend(String variantBackend) + { + processMetaData.setVariantBackend(variantBackend); + return (this); + } } } From c5e381abdb235fde499009eeb7da5eea9a322b69 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Mar 2024 10:53:18 -0500 Subject: [PATCH 46/72] Add toString --- .../backend/core/logging/CollectedLogMessage.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java index ad96eac5..ff607b8e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java @@ -48,6 +48,21 @@ public class CollectedLogMessage + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return "CollectedLogMessage{" + + "level=" + level + + ", message='" + message + '\'' + + ", exception=" + exception + + '}'; + } + + + /******************************************************************************* ** Getter for message *******************************************************************************/ From 5ed21d1fed16067bb11493ce730280b066a458c6 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Mar 2024 10:55:42 -0500 Subject: [PATCH 47/72] Add memoization of constructors in getAdHoc (micro optimization) --- .../core/actions/customizers/QCodeLoader.java | 38 ++++---- .../actions/customizers/QCodeLoaderTest.java | 94 +++++++++++++++++++ 2 files changed, 116 insertions(+), 16 deletions(-) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoaderTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java index 23162753..a8adaed6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.actions.customizers; +import java.lang.reflect.Constructor; import java.util.Optional; import java.util.function.Function; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; @@ -34,30 +35,23 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* ** Utility to load code for running QQQ customizers. + ** + ** TODO - redo all to go through method that memoizes class & constructor + ** lookup. That memoziation causes 1,000,000 such calls to go from ~500ms + ** to ~100ms. *******************************************************************************/ public class QCodeLoader { private static final QLogger LOG = QLogger.getLogger(QCodeLoader.class); - - - /******************************************************************************* - ** - *******************************************************************************/ - public static Optional> getTableCustomizerFunction(QTableMetaData table, String customizerName) - { - Optional codeReference = table.getCustomizer(customizerName); - if(codeReference.isPresent()) - { - return (Optional.ofNullable(QCodeLoader.getFunction(codeReference.get()))); - } - return (Optional.empty()); - } + private static Memoization> constructorMemoization = new Memoization<>(); @@ -175,8 +169,20 @@ public class QCodeLoader try { - Class customizerClass = Class.forName(codeReference.getName()); - return ((T) customizerClass.getConstructor().newInstance()); + Optional> constructor = constructorMemoization.getResultThrowing(codeReference.getName(), (UnsafeFunction, Exception>) s -> { + Class customizerClass = Class.forName(codeReference.getName()); + return customizerClass.getConstructor(); + }); + + if(constructor.isPresent()) + { + return ((T) constructor.get().newInstance()); + } + else + { + LOG.error("Could not get constructor for code reference", logPair("codeReference", codeReference)); + return (null); + } } catch(Exception e) { diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoaderTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoaderTest.java new file mode 100644 index 00000000..83cf8298 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoaderTest.java @@ -0,0 +1,94 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.customizers; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.utils.Timer; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for QCodeLoader + *******************************************************************************/ +class QCodeLoaderTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetAdHoc() + { + QCodeLoader qCodeLoader = QCodeLoader.getAdHoc(QCodeLoader.class, new QCodeReference(QCodeLoader.class)); + assertThat(qCodeLoader).isInstanceOf(QCodeLoader.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + @Disabled("performance test, used during memoization change") + void testBulkPerformance() + { + Timer timer = new Timer("start"); + for(int i = 0; i < 5; i++) + { + useCodeLoader(1_000_000); + timer.mark("done with code loader"); + + useNew(1_000_000); + timer.mark("done with new"); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void useNew(int count) + { + for(int i = 0; i < count; i++) + { + QCodeLoader qCodeLoader = new QCodeLoader(); + } + } + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void useCodeLoader(int count) + { + for(int i = 0; i < count; i++) + { + QCodeLoader qCodeLoader = QCodeLoader.getAdHoc(QCodeLoader.class, new QCodeReference(QCodeLoader.class)); + } + } + +} \ No newline at end of file From 0130e34112577fea70c1208a2d15c1c289980937 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Mar 2024 10:56:37 -0500 Subject: [PATCH 48/72] Change withTemporaryContext to take UnsafeVoidVoidMethod instead of VoidVoidMethod --- .../qqq/backend/core/context/QContext.java | 4 +- .../utils/lambdas/UnsafeVoidVoidMethod.java | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/lambdas/UnsafeVoidVoidMethod.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java index c0ca377c..cfc931b4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java @@ -34,7 +34,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.session.QSession; -import com.kingsrook.qqq.backend.core.utils.lambdas.VoidVoidMethod; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeVoidVoidMethod; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -110,7 +110,7 @@ public class QContext /******************************************************************************* ** *******************************************************************************/ - public static void withTemporaryContext(CapturedContext context, VoidVoidMethod method) + public static void withTemporaryContext(CapturedContext context, UnsafeVoidVoidMethod method) throws T { CapturedContext originalContext = QContext.capture(); try diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/lambdas/UnsafeVoidVoidMethod.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/lambdas/UnsafeVoidVoidMethod.java new file mode 100644 index 00000000..e448d2e9 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/lambdas/UnsafeVoidVoidMethod.java @@ -0,0 +1,37 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.lambdas; + + +/******************************************************************************* + ** + *******************************************************************************/ +@FunctionalInterface +public interface UnsafeVoidVoidMethod +{ + + /******************************************************************************* + ** + *******************************************************************************/ + void run() throws T; + +} From 753c22419688f84a4d0b9703e04636aa6fa76aac Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Mar 2024 12:24:40 -0500 Subject: [PATCH 49/72] CE-936 - Refactored, for more generic handling of job types - Introduced SchedulableType, SchedulableRunner, SchedulableIdentity classes - removed job-type specific methods from QScheduleManager and QSchedulerInterface - Add scheduler-level management processes - Change quartz to not change schedules during service startup - re-added repeatSeconds to ScheduledJob --- .../core/instances/QInstanceValidator.java | 7 +- .../core/model/metadata/QInstance.java | 55 ++- .../model/scheduledjobs/ScheduledJob.java | 35 ++ .../ScheduledJobsMetaDataProvider.java | 14 +- .../core/scheduler/QScheduleManager.java | 343 ++++++++++-------- .../core/scheduler/QSchedulerInterface.java | 61 ++-- .../core/scheduler/SchedulerUtils.java | 42 +-- .../processes/RescheduleAllJobsProcess.java | 90 +++++ .../processes/UnscheduleAllJobsProcess.java | 90 +++++ ...SqsPollerJob.java => QuartzJobRunner.java} | 45 +-- .../scheduler/quartz/QuartzRunProcessJob.java | 93 ----- .../scheduler/quartz/QuartzScheduler.java | 188 +++++++--- .../quartz/QuartzTableAutomationsJob.java | 104 ------ .../processes/PauseAllQuartzJobsProcess.java | 6 +- .../processes/ResumeAllQuartzJobsProcess.java | 6 +- .../schedulable/SchedulableType.java | 98 +++++ .../identity/BasicSchedulableIdentity.java | 121 ++++++ .../identity/SchedulableIdentity.java | 55 +++ .../identity/SchedulableIdentityFactory.java | 96 +++++ .../runner/SchedulableProcessRunner.java | 188 ++++++++++ .../schedulable/runner/SchedulableRunner.java | 51 +++ .../runner/SchedulableSQSQueueRunner.java | 135 +++++++ .../SchedulableTableAutomationsRunner.java | 168 +++++++++ .../scheduler/simple/SimpleJobRunner.java | 87 +++++ .../scheduler/simple/SimpleScheduler.java | 141 +++---- .../core/scheduler/QScheduleManagerTest.java | 205 +++++++++++ .../core/scheduler/SchedulerTestUtils.java | 79 ++++ .../scheduler/quartz/QuartzSchedulerTest.java | 85 ++--- .../processes/QuartzJobsProcessTest.java | 1 + .../scheduler/simple/SimpleSchedulerTest.java | 43 +-- 30 files changed, 2059 insertions(+), 673 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcess.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcess.java rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/{QuartzSqsPollerJob.java => QuartzJobRunner.java} (52%) delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/SchedulableType.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/BasicSchedulableIdentity.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentity.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableProcessRunner.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableRunner.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableSQSQueueRunner.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 43e1814e..a30f1a1d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -435,10 +435,7 @@ public class QInstanceValidator assertCondition(qInstance.getProcesses() != null && qInstance.getProcess(queue.getProcessName()) != null, "Unrecognized processName for queue: " + name); } - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // todo - if we have, in the future, a provider that doesn't require schedules per-queue, then make this check conditional // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(assertCondition(queue.getSchedule() != null, "Missing schedule for SQSQueueProvider: " + name)) + if(queue.getSchedule() != null) { validateScheduleMetaData(queue.getSchedule(), qInstance, "SQSQueueProvider " + name + ", schedule: "); } @@ -1024,7 +1021,7 @@ public class QInstanceValidator assertCondition(qInstance.getAutomationProvider(providerName) != null, " has an unrecognized providerName: " + providerName); } - if(assertCondition(automationDetails.getSchedule() != null, prefix + "Missing schedule for automations")) + if(automationDetails.getSchedule() != null) { validateScheduleMetaData(automationDetails.getSchedule(), qInstance, prefix + " automationDetails, schedule: "); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 5a5b4ac3..255e4c7e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -56,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import io.github.cdimascio.dotenv.Dotenv; @@ -91,7 +92,9 @@ public class QInstance private Map widgets = new LinkedHashMap<>(); private Map queueProviders = new LinkedHashMap<>(); private Map queues = new LinkedHashMap<>(); - private Map schedulers = new LinkedHashMap<>(); + + private Map schedulers = new LinkedHashMap<>(); + private Map schedulableTypes = new LinkedHashMap<>(); private Map supplementalMetaData = new LinkedHashMap<>(); @@ -1278,4 +1281,54 @@ public class QInstance } + + /******************************************************************************* + ** + *******************************************************************************/ + public void addSchedulableType(SchedulableType schedulableType) + { + String name = schedulableType.getName(); + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a schedulableType without a name.")); + } + if(this.schedulableTypes.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second schedulableType with name: " + name)); + } + this.schedulableTypes.put(name, schedulableType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public SchedulableType getSchedulableType(String name) + { + return (this.schedulableTypes.get(name)); + } + + + + /******************************************************************************* + ** Getter for schedulableTypes + ** + *******************************************************************************/ + public Map getSchedulableTypes() + { + return schedulableTypes; + } + + + + /******************************************************************************* + ** Setter for schedulableTypes + ** + *******************************************************************************/ + public void setSchedulableTypes(Map schedulableTypes) + { + this.schedulableTypes = schedulableTypes; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java index 487252b4..54bfc182 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java @@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.data.QAssociation; import com.kingsrook.qqq.backend.core.model.data.QField; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.collections.MutableMap; @@ -69,6 +70,9 @@ public class ScheduledJob extends QRecordEntity @QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = TimeZonePossibleValueSourceMetaDataProvider.NAME) private String cronTimeZoneId; + @QField(displayFormat = DisplayFormat.COMMAS) + private Integer repeatSeconds; + @QField(isRequired = true, maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = ScheduledJobType.NAME) private String type; @@ -458,4 +462,35 @@ public class ScheduledJob extends QRecordEntity return (this); } + + /******************************************************************************* + ** Getter for repeatSeconds + *******************************************************************************/ + public Integer getRepeatSeconds() + { + return (this.repeatSeconds); + } + + + + /******************************************************************************* + ** Setter for repeatSeconds + *******************************************************************************/ + public void setRepeatSeconds(Integer repeatSeconds) + { + this.repeatSeconds = repeatSeconds; + } + + + + /******************************************************************************* + ** Fluent setter for repeatSeconds + *******************************************************************************/ + public ScheduledJob withRepeatSeconds(Integer repeatSeconds) + { + this.repeatSeconds = repeatSeconds; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java index 92d0e22f..352b5de1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java @@ -66,7 +66,7 @@ public class ScheduledJobsMetaDataProvider instance.addPossibleValueSource(QPossibleValueSource.newForEnum(ScheduledJobType.NAME, ScheduledJobType.values())); instance.addPossibleValueSource(defineSchedulersPossibleValueSource()); defineStandardJoins(instance); - defineStandardScriptsWidgets(instance); + defineStandardWidgets(instance); } @@ -74,11 +74,12 @@ public class ScheduledJobsMetaDataProvider /******************************************************************************* ** *******************************************************************************/ - public void defineStandardScriptsWidgets(QInstance instance) + public void defineStandardWidgets(QInstance instance) { QJoinMetaData join = instance.getJoin(JOB_PARAMETER_JOIN_NAME); instance.addWidget(ChildRecordListRenderer.widgetMetaDataBuilder(join) .withCanAddChildRecord(true) + .withManageAssociationName(ScheduledJobParameter.TABLE_NAME) .withLabel("Parameters") .getWidgetMetaData() .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED))); @@ -165,7 +166,7 @@ public class ScheduledJobsMetaDataProvider .withRecordLabelFormat("%s") .withRecordLabelFields("label") .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "description"))) - .withSection(new QFieldSection("schedule", new QIcon().withName("alarm"), Tier.T2, List.of("cronExpression", "cronTimeZoneId"))) + .withSection(new QFieldSection("schedule", new QIcon().withName("alarm"), Tier.T2, List.of("cronExpression", "cronTimeZoneId", "repeatSeconds"))) .withSection(new QFieldSection("settings", new QIcon().withName("tune"), Tier.T2, List.of("type", "isActive", "schedulerName"))) .withSection(new QFieldSection("parameters", new QIcon().withName("list"), Tier.T2).withWidgetName(JOB_PARAMETER_JOIN_NAME)) .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); @@ -178,9 +179,9 @@ public class ScheduledJobsMetaDataProvider tableMetaData.withCustomizer(TableCustomizers.POST_DELETE_RECORD, customizerReference); tableMetaData.withAssociation(new Association() + .withName(ScheduledJobParameter.TABLE_NAME) .withAssociatedTableName(ScheduledJobParameter.TABLE_NAME) - .withJoinName(JOB_PARAMETER_JOIN_NAME) - .withName(ScheduledJobParameter.TABLE_NAME)); + .withJoinName(JOB_PARAMETER_JOIN_NAME)); return (tableMetaData); } @@ -195,8 +196,7 @@ public class ScheduledJobsMetaDataProvider QTableMetaData tableMetaData = defineStandardTable(backendName, ScheduledJobParameter.TABLE_NAME, ScheduledJobParameter.class) .withRecordLabelFormat("%s - %s") .withRecordLabelFields("scheduledJobId", "key") - .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "scheduledJobId", "key"))) - .withSection(new QFieldSection("value", new QIcon().withName("dataset"), Tier.T2, List.of("value"))) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "scheduledJobId", "key", "value"))) .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); return (tableMetaData); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index da9ca9b8..747c366b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.CapturedContext; import com.kingsrook.qqq.backend.core.context.QContext; @@ -40,19 +41,27 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.VariantRunStrategy; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QSchedulerMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentityFactory; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableProcessRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableSQSQueueRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableTableAutomationsRunner; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; -import org.apache.commons.lang.NotImplementedException; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -87,17 +96,48 @@ public class QScheduleManager ** Singleton initiator - e.g., must be called to initially initialize the singleton ** before anyone else calls getInstance (they'll get an error if they call that first). *******************************************************************************/ - public static QScheduleManager initInstance(QInstance qInstance, Supplier systemUserSessionSupplier) + public static QScheduleManager initInstance(QInstance qInstance, Supplier systemUserSessionSupplier) throws QException { if(qScheduleManager == null) { qScheduleManager = new QScheduleManager(qInstance, systemUserSessionSupplier); + + ///////////////////////////////////////////////////////////////// + // if the instance doesn't have any schedulable types defined, // + // then go ahead and add the default set that qqq knows about // + ///////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeIsEmpty(qInstance.getSchedulableTypes())) + { + defineDefaultSchedulableTypesInInstance(qInstance); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // initialize the scheduler(s) we're configured to use // + // do this, even if we won't start them - so, for example, a web server can still be aware of schedules in the application // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QSchedulerMetaData schedulerMetaData : CollectionUtils.nonNullMap(qInstance.getSchedulers()).values()) + { + QSchedulerInterface scheduler = schedulerMetaData.initSchedulerInstance(qInstance, systemUserSessionSupplier); + qScheduleManager.schedulers.put(schedulerMetaData.getName(), scheduler); + } } return (qScheduleManager); } + /******************************************************************************* + ** + *******************************************************************************/ + public static void defineDefaultSchedulableTypesInInstance(QInstance qInstance) + { + qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.PROCESS.getId()).withRunner(new QCodeReference(SchedulableProcessRunner.class))); + qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.QUEUE_PROCESSOR.getId()).withRunner(new QCodeReference(SchedulableSQSQueueRunner.class))); + qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.TABLE_AUTOMATIONS.getId()).withRunner(new QCodeReference(SchedulableTableAutomationsRunner.class))); + } + + + /******************************************************************************* ** Singleton accessor *******************************************************************************/ @@ -117,29 +157,20 @@ public class QScheduleManager *******************************************************************************/ public void start() throws QException { - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // initialize the scheduler(s) we're configured to use // - // do this, even if we won't start them - so, for example, a web server can still be aware of schedules in the application // - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - for(QSchedulerMetaData schedulerMetaData : CollectionUtils.nonNullMap(qInstance.getSchedulers()).values()) - { - QSchedulerInterface scheduler = schedulerMetaData.initSchedulerInstance(qInstance, systemUserSessionSupplier); - schedulers.put(schedulerMetaData.getName(), scheduler); - } - - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - // now, exist w/o setting up schedules and not starting schedules, if schedule manager isn't enabled here // - //////////////////////////////////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + // exit w/o starting schedulers, if schedule manager isn't enabled here // + ////////////////////////////////////////////////////////////////////////// if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) { LOG.info("Not starting ScheduleManager per settings."); + schedulers.values().forEach(s -> s.doNotStart()); return; } ///////////////////////////////////////////////////////////////////////////////////////////////// // ensure that everything which should be scheduled is scheduled, in the appropriate scheduler // ///////////////////////////////////////////////////////////////////////////////////////////////// - QContext.withTemporaryContext(new CapturedContext(qInstance, systemUserSessionSupplier.get()), () -> setupSchedules()); + QContext.withTemporaryContext(new CapturedContext(qInstance, systemUserSessionSupplier.get()), () -> setupAllSchedules()); ////////////////////////// // start each scheduler // @@ -172,7 +203,7 @@ public class QScheduleManager /******************************************************************************* ** *******************************************************************************/ - private void setupSchedules() + public void setupAllSchedules() throws QException { ///////////////////////////////////////////// // read dynamic schedules // @@ -199,20 +230,27 @@ public class QScheduleManager ///////////////////////////////////////////////////////// schedulers.values().forEach(s -> s.startOfSetupSchedules()); - ////////////////////////////////// - // schedule all queue providers // - ////////////////////////////////// - for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values()) + ///////////////////////// + // schedule all queues // + ///////////////////////// + for(QQueueMetaData queue : qInstance.getQueues().values()) { - setupQueueProvider(queueProvider); + if(queue.getSchedule() != null) + { + setupQueue(queue); + } } - /////////////////////////////////////// - // schedule all automation providers // - /////////////////////////////////////// - for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values()) + //////////////////////////////////////// + // schedule all tables w/ automations // + //////////////////////////////////////// + for(QTableMetaData table : qInstance.getTables().values()) { - setupAutomationProviderPerTable(automationProvider); + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + if(automationDetails != null && automationDetails.getSchedule() != null) + { + setupTableAutomations(table); + } } ///////////////////////////////////////// @@ -253,47 +291,60 @@ public class QScheduleManager /******************************************************************************* ** *******************************************************************************/ - public void setupScheduledJob(ScheduledJob scheduledJob) + public void setupScheduledJob(ScheduledJob scheduledJob) throws QException { - /////////////////////////////////////////////////////////////////////////////////////////// - // non-active jobs should be deleted from the scheduler. they get re-added // - // if they get re-activated. but we don't want to rely on (e.g., for quartz) the paused // - // state to be drive by is-active. else, devops-pause & unpause ops would clobber // - // scheduled-job record facts // - /////////////////////////////////////////////////////////////////////////////////////////// + BasicSchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(scheduledJob); + + //////////////////////////////////////////////////////////////////////////////// + // non-active jobs should be deleted from the scheduler. they get re-added // + // if they get re-activated. but we don't want to rely on (e.g., for quartz) // + // the paused state to be drive by is-active. else, devops-pause & unpause // + // operations would clobber scheduled-job record facts // + //////////////////////////////////////////////////////////////////////////////// if(!scheduledJob.getIsActive()) { unscheduleScheduledJob(scheduledJob); return; } - QSchedulerInterface scheduler = getScheduler(scheduledJob.getSchedulerName()); + String exceptionSuffix = "in scheduledJob [" + scheduledJob.getId() + "]"; + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // setup schedule meta-data object based on schedule data in the scheduled job - throwing if not well populated // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(scheduledJob.getRepeatSeconds() == null && !StringUtils.hasContent(scheduledJob.getCronExpression())) + { + throw (new QException("Missing a schedule (cronString or repeatSeconds) " + exceptionSuffix)); + } QScheduleMetaData scheduleMetaData = new QScheduleMetaData(); scheduleMetaData.setCronExpression(scheduledJob.getCronExpression()); scheduleMetaData.setCronTimeZoneId(scheduledJob.getCronTimeZoneId()); + scheduleMetaData.setRepeatSeconds(scheduledJob.getRepeatSeconds()); - switch(ScheduledJobType.getById(scheduledJob.getType())) + ///////////////////////////////// + // get & validate the job type // + ///////////////////////////////// + if(!StringUtils.hasContent(scheduledJob.getType())) { - case PROCESS -> - { - Map paramMap = scheduledJob.getJobParametersMap(); - String processName = paramMap.get("processName"); - QProcessMetaData process = qInstance.getProcess(processName); - - // todo - variants... serial vs parallel? - scheduler.setupProcess(process, null, scheduleMetaData, true); - } - case QUEUE_PROCESSOR -> - { - throw new NotImplementedException("ScheduledJob queue processors are not yet implemented..."); - } - case TABLE_AUTOMATIONS -> - { - throw new NotImplementedException("ScheduledJob table automations are not yet implemented..."); - } - default -> throw new IllegalStateException("Unexpected value: " + ScheduledJobType.getById(scheduledJob.getType())); + throw (new QException("Missing a type " + exceptionSuffix)); } + + ScheduledJobType scheduledJobType = ScheduledJobType.getById(scheduledJob.getType()); + if(scheduledJobType == null) + { + throw (new QException("Unrecognized type [" + scheduledJob.getType() + "] " + exceptionSuffix)); + } + + QSchedulerInterface scheduler = getScheduler(scheduledJob.getSchedulerName()); + Map paramMap = new HashMap<>(scheduledJob.getJobParametersMap()); + + SchedulableType schedulableType = qInstance.getSchedulableType(scheduledJob.getType()); + + SchedulableRunner runner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + runner.validateParams(schedulableIdentity, new HashMap<>(paramMap)); + + scheduler.setupSchedulable(schedulableIdentity, schedulableType, paramMap, scheduleMetaData, true); } @@ -301,29 +352,34 @@ public class QScheduleManager /******************************************************************************* ** *******************************************************************************/ - public void unscheduleScheduledJob(ScheduledJob scheduledJob) + public void unscheduleAll() + { + schedulers.values().forEach(s -> + { + try + { + s.unscheduleAll(); + } + catch(Exception e) + { + LOG.warn("Error unscheduling everything in scheduler " + s, e); + } + }); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void unscheduleScheduledJob(ScheduledJob scheduledJob) throws QException { QSchedulerInterface scheduler = getScheduler(scheduledJob.getSchedulerName()); - switch(ScheduledJobType.getById(scheduledJob.getType())) - { - case PROCESS -> - { - Map paramMap = scheduledJob.getJobParametersMap(); - String processName = paramMap.get("processName"); - QProcessMetaData process = qInstance.getProcess(processName); - scheduler.unscheduleProcess(process); - } - case QUEUE_PROCESSOR -> - { - throw new NotImplementedException("ScheduledJob queue processors are not yet implemented..."); - } - case TABLE_AUTOMATIONS -> - { - throw new NotImplementedException("ScheduledJob table automations are not yet implemented..."); - } - default -> throw new IllegalStateException("Unexpected value: " + ScheduledJobType.getById(scheduledJob.getType())); - } + BasicSchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(scheduledJob); + SchedulableType schedulableType = qInstance.getSchedulableType(scheduledJob.getType()); + + scheduler.unscheduleSchedulable(schedulableIdentity, schedulableType); } @@ -331,28 +387,45 @@ public class QScheduleManager /******************************************************************************* ** *******************************************************************************/ - private void setupProcess(QProcessMetaData process) + private void setupProcess(QProcessMetaData process) throws QException { - QScheduleMetaData scheduleMetaData = process.getSchedule(); - if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) + BasicSchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(process); + QSchedulerInterface scheduler = getScheduler(process.getSchedule().getSchedulerName()); + boolean allowedToStart = SchedulerUtils.allowedToStart(process.getName()); + + Map paramMap = new HashMap<>(); + paramMap.put("processName", process.getName()); + + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.PROCESS.getId()); + + if(process.getVariantBackend() == null || VariantRunStrategy.SERIAL.equals(process.getVariantRunStrategy())) { /////////////////////////////////////////////// // if no variants, or variant is serial mode // /////////////////////////////////////////////// - setupProcess(process, null); + scheduler.setupSchedulable(schedulableIdentity, schedulableType, new HashMap<>(paramMap), process.getSchedule(), allowedToStart); } - else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) + else if(VariantRunStrategy.PARALLEL.equals(process.getVariantRunStrategy())) { ///////////////////////////////////////////////////////////////////////////////////////////////////// // if this a "parallel", which for example means we want to have a thread for each backend variant // // running at the same time, get the variant records and schedule each separately // ///////////////////////////////////////////////////////////////////////////////////////////////////// - QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); + QBackendMetaData backendMetaData = qInstance.getBackend(process.getVariantBackend()); for(QRecord qRecord : CollectionUtils.nonNullList(SchedulerUtils.getBackendVariantFilteredRecords(process))) { try { - setupProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); + HashMap parameters = new HashMap<>(paramMap); + HashMap variantMap = new HashMap<>(Map.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); + parameters.put("backendVariantData", variantMap); + + String identity = schedulableIdentity.getIdentity() + ";" + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()); + String description = schedulableIdentity.getDescription() + " for variant: " + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()); + + BasicSchedulableIdentity variantIdentity = new BasicSchedulableIdentity(identity, description); + + scheduler.setupSchedulable(variantIdentity, schedulableType, parameters, process.getSchedule(), allowedToStart); } catch(Exception e) { @@ -362,7 +435,7 @@ public class QScheduleManager } else { - LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided."); + LOG.error("Unsupported Schedule Run Strategy [" + process.getVariantRunStrategy() + "] was provided."); } } @@ -371,71 +444,25 @@ public class QScheduleManager /******************************************************************************* ** *******************************************************************************/ - private void setupProcess(QProcessMetaData process, Map backendVariantData) + private void setupTableAutomations(QTableMetaData table) throws QException { - QSchedulerInterface scheduler = getScheduler(process.getSchedule().getSchedulerName()); - scheduler.setupProcess(process, backendVariantData, process.getSchedule(), SchedulerUtils.allowedToStart(process)); - } + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.TABLE_AUTOMATIONS.getId()); + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + QSchedulerInterface scheduler = getScheduler(automationDetails.getSchedule().getSchedulerName()); + List tableActionList = PollingAutomationPerTableRunner.getTableActions(qInstance, automationDetails.getProviderName()) + .stream().filter(ta -> ta.tableName().equals(table.getName())) + .toList(); - - /******************************************************************************* - ** - *******************************************************************************/ - private void setupQueueProvider(QQueueProviderMetaData queueProvider) - { - switch(queueProvider.getType()) - { - case SQS: - setupSqsProvider((SQSQueueProviderMetaData) queueProvider); - break; - default: - throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType()); - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void setupSqsProvider(SQSQueueProviderMetaData queueProvider) - { - boolean allowedToStartProvider = SchedulerUtils.allowedToStart(queueProvider); - - for(QQueueMetaData queue : qInstance.getQueues().values()) - { - QSchedulerInterface scheduler = getScheduler(queue.getSchedule().getSchedulerName()); - - boolean allowedToStart = allowedToStartProvider && SchedulerUtils.allowedToStart(queue.getName()); - if(queueProvider.getName().equals(queue.getProviderName())) - { - scheduler.setupSqsPoller(queueProvider, queue, queue.getSchedule(), allowedToStart); - } - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void setupAutomationProviderPerTable(QAutomationProviderMetaData automationProvider) - { - boolean allowedToStartProvider = SchedulerUtils.allowedToStart(automationProvider); - - /////////////////////////////////////////////////////////////////////////////////// - // ask the PollingAutomationPerTableRunner how many threads of itself need setup // - // then schedule each one of them. // - /////////////////////////////////////////////////////////////////////////////////// - List tableActionList = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName()); for(PollingAutomationPerTableRunner.TableActionsInterface tableActions : tableActionList) { - boolean allowedToStart = allowedToStartProvider && SchedulerUtils.allowedToStart(tableActions.tableName()); + SchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(tableActions); + boolean allowedToStart = SchedulerUtils.allowedToStart(table.getName()); - QScheduleMetaData schedule = tableActions.tableAutomationDetails().getSchedule(); - QSchedulerInterface scheduler = getScheduler(schedule.getSchedulerName()); - scheduler.setupTableAutomation(automationProvider, tableActions, schedule, allowedToStart); + Map paramMap = new HashMap<>(); + paramMap.put("tableName", tableActions.tableName()); + paramMap.put("automationStatus", tableActions.status().name()); + scheduler.setupSchedulable(schedulableIdentity, schedulableType, new HashMap<>(paramMap), automationDetails.getSchedule(), allowedToStart); } } @@ -444,12 +471,34 @@ public class QScheduleManager /******************************************************************************* ** *******************************************************************************/ - private QSchedulerInterface getScheduler(String schedulerName) + private void setupQueue(QQueueMetaData queue) throws QException { + SchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(queue); + QSchedulerInterface scheduler = getScheduler(queue.getSchedule().getSchedulerName()); + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.QUEUE_PROCESSOR.getId()); + boolean allowedToStart = SchedulerUtils.allowedToStart(queue.getName()); + + Map paramMap = new HashMap<>(); + paramMap.put("queueName", queue.getName()); + scheduler.setupSchedulable(schedulableIdentity, schedulableType, new HashMap<>(paramMap), queue.getSchedule(), allowedToStart); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QSchedulerInterface getScheduler(String schedulerName) throws QException + { + if(!StringUtils.hasContent(schedulerName)) + { + throw (new QException("Scheduler name was not given (and the concept of a default scheduler does not exist at this time).")); + } + QSchedulerInterface scheduler = schedulers.get(schedulerName); if(scheduler == null) { - throw new NotImplementedException("default scheduler..."); + throw (new QException("Unrecognized schedulerName [" + schedulerName + "]")); } return (scheduler); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java index 1e18d863..b9f16902 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QSchedulerInterface.java @@ -24,12 +24,10 @@ package com.kingsrook.qqq.backend.core.scheduler; import java.io.Serializable; import java.util.Map; -import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; -import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; /******************************************************************************* @@ -37,31 +35,41 @@ import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaDa *******************************************************************************/ public interface QSchedulerInterface { - /******************************************************************************* - ** - *******************************************************************************/ - void setupProcess(QProcessMetaData process, Map backendVariantData, QScheduleMetaData schedule, boolean allowedToStart); /******************************************************************************* ** *******************************************************************************/ - void setupSqsPoller(SQSQueueProviderMetaData queueProvider, QQueueMetaData queue, QScheduleMetaData schedule, boolean allowedToStart); - - /******************************************************************************* - ** - *******************************************************************************/ - void setupTableAutomation(QAutomationProviderMetaData automationProvider, PollingAutomationPerTableRunner.TableActionsInterface tableActions, QScheduleMetaData schedule, boolean allowedToStart); - - /******************************************************************************* - ** - *******************************************************************************/ - void unscheduleProcess(QProcessMetaData process); + String getSchedulerName(); /******************************************************************************* ** *******************************************************************************/ void start(); + /******************************************************************************* + ** called to indicate that the schedule manager is past its startup routine, + ** but that the schedule should not actually be running in this process. + *******************************************************************************/ + default void doNotStart() + { + + } + + /******************************************************************************* + ** + *******************************************************************************/ + void setupSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType, Map parameters, QScheduleMetaData schedule, boolean allowedToStart); + + /******************************************************************************* + ** + *******************************************************************************/ + void unscheduleSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType); + + /******************************************************************************* + ** + *******************************************************************************/ + void unscheduleAll() throws QException; + /******************************************************************************* ** *******************************************************************************/ @@ -77,7 +85,9 @@ public interface QSchedulerInterface *******************************************************************************/ default void unInit() { - + ///////////////////// + // noop by default // + ///////////////////// } /******************************************************************************* @@ -85,7 +95,9 @@ public interface QSchedulerInterface *******************************************************************************/ default void startOfSetupSchedules() { - + ///////////////////// + // noop by default // + ///////////////////// } /******************************************************************************* @@ -93,6 +105,9 @@ public interface QSchedulerInterface *******************************************************************************/ default void endOfSetupSchedules() { - + ///////////////////// + // noop by default // + ///////////////////// } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java index e44c124d..6feeb7ee 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java @@ -41,10 +41,10 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.VariantRunStrategy; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; @@ -56,17 +56,6 @@ public class SchedulerUtils private static final QLogger LOG = QLogger.getLogger(SchedulerUtils.class); - - /******************************************************************************* - ** - *******************************************************************************/ - public static boolean allowedToStart(TopLevelMetaDataInterface metaDataObject) - { - return (allowedToStart(metaDataObject.getName())); - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -87,7 +76,7 @@ public class SchedulerUtils /******************************************************************************* ** *******************************************************************************/ - public static void runProcess(QInstance qInstance, Supplier sessionSupplier, QProcessMetaData process, Map backendVariantData) + public static void runProcess(QInstance qInstance, Supplier sessionSupplier, QProcessMetaData process, Map backendVariantData, Map processInputValues) { String originalThreadName = Thread.currentThread().getName(); @@ -95,11 +84,11 @@ public class SchedulerUtils { QContext.init(qInstance, sessionSupplier.get()); - if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy())) + if(process.getVariantBackend() == null || VariantRunStrategy.PARALLEL.equals(process.getVariantRunStrategy())) { - SchedulerUtils.executeSingleProcess(process, backendVariantData); + executeSingleProcess(process, backendVariantData, processInputValues); } - else if(QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy())) + else if(VariantRunStrategy.SERIAL.equals(process.getVariantRunStrategy())) { /////////////////////////////////////////////////////////////////////////////////////////////////// // if this is "serial", which for example means we want to run each backend variant one after // @@ -109,9 +98,9 @@ public class SchedulerUtils { try { - QScheduleMetaData scheduleMetaData = process.getSchedule(); - QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend()); - executeSingleProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()))); + QBackendMetaData backendMetaData = qInstance.getBackend(process.getVariantBackend()); + Map thisVariantData = MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())); + executeSingleProcess(process, thisVariantData, processInputValues); } catch(Exception e) { @@ -136,7 +125,7 @@ public class SchedulerUtils /******************************************************************************* ** *******************************************************************************/ - private static void executeSingleProcess(QProcessMetaData process, Map backendVariantData) throws QException + private static void executeSingleProcess(QProcessMetaData process, Map backendVariantData, Map processInputValues) throws QException { if(backendVariantData != null) { @@ -144,10 +133,16 @@ public class SchedulerUtils } Thread.currentThread().setName("ScheduledProcess>" + process.getName()); - LOG.debug("Running Scheduled Process [" + process.getName() + "]"); + LOG.debug("Running Scheduled Process [" + process.getName() + "] with values [" + processInputValues + "]"); RunProcessInput runProcessInput = new RunProcessInput(); runProcessInput.setProcessName(process.getName()); + + for(Map.Entry entry : CollectionUtils.nonNullMap(processInputValues).entrySet()) + { + runProcessInput.withValue(entry.getKey(), entry.getValue()); + } + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); QContext.pushAction(runProcessInput); @@ -166,8 +161,7 @@ public class SchedulerUtils List records = null; try { - QScheduleMetaData scheduleMetaData = processMetaData.getSchedule(); - QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(scheduleMetaData.getVariantBackend()); + QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(processMetaData.getVariantBackend()); QueryInput queryInput = new QueryInput(); queryInput.setTableName(backendMetaData.getVariantOptionsTableName()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcess.java new file mode 100644 index 00000000..9c3840ca --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/RescheduleAllJobsProcess.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; + + +/******************************************************************************* + ** Management process to reschedule all scheduled jobs (in all schedulers). + *******************************************************************************/ +public class RescheduleAllJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Reschedule all Scheduled Jobs") + .withIcon(new QIcon("update")) + .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to reschedule all jobs."))), + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All jobs have been rescheduled."))))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QScheduleManager.getInstance().setupAllSchedules(); + } + catch(Exception e) + { + throw (new QException("Error setting up all scheduled jobs.", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcess.java new file mode 100644 index 00000000..11492b06 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/processes/UnscheduleAllJobsProcess.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.processes; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; + + +/******************************************************************************* + ** Management process to unschedule all scheduled jobs (in all schedulers). + *******************************************************************************/ +public class UnscheduleAllJobsProcess implements BackendStep, MetaDataProducerInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QProcessMetaData produce(QInstance qInstance) throws QException + { + return new QProcessMetaData() + .withName(getClass().getSimpleName()) + .withLabel("Unschedule all Scheduled Jobs") + .withIcon(new QIcon("update_disabled")) + .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to unschedule all jobs."))), + new QBackendStepMetaData() + .withName("execute") + .withCode(new QCodeReference(getClass())), + new QFrontendStepMetaData() + .withName("results") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("All jobs have been unscheduled."))))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + QScheduleManager.getInstance().unscheduleAll(); + } + catch(Exception e) + { + throw (new QException("Error unscheduling all scheduled jobs.", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java similarity index 52% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java index 140a90d4..c76b7d60 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSqsPollerJob.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java @@ -22,14 +22,15 @@ package com.kingsrook.qqq.backend.core.scheduler.quartz; -import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.context.CapturedContext; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; -import org.quartz.DisallowConcurrentExecution; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; import org.quartz.Job; -import org.quartz.JobDataMap; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -38,10 +39,9 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* ** *******************************************************************************/ -@DisallowConcurrentExecution -public class QuartzSqsPollerJob implements Job +public class QuartzJobRunner implements Job { - private static final QLogger LOG = QLogger.getLogger(QuartzSqsPollerJob.class); + private static final QLogger LOG = QLogger.getLogger(QuartzJobRunner.class); @@ -49,37 +49,28 @@ public class QuartzSqsPollerJob implements Job ** *******************************************************************************/ @Override - public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException + public void execute(JobExecutionContext context) throws JobExecutionException { - String queueProviderName = null; - String queueName = null; - + CapturedContext capturedContext = QContext.capture(); try { - JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); - queueProviderName = jobDataMap.getString("queueProviderName"); - queueName = jobDataMap.getString("queueName"); - QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + QuartzScheduler quartzScheduler = QuartzScheduler.getInstance(); + QInstance qInstance = quartzScheduler.getQInstance(); + QContext.init(qInstance, quartzScheduler.getSessionSupplier().get()); - SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); - sqsQueuePoller.setQueueProviderMetaData((SQSQueueProviderMetaData) qInstance.getQueueProvider(queueProviderName)); - sqsQueuePoller.setQueueMetaData(qInstance.getQueue(queueName)); - sqsQueuePoller.setQInstance(qInstance); - sqsQueuePoller.setSessionSupplier(QuartzScheduler.getInstance().getSessionSupplier()); + SchedulableType schedulableType = qInstance.getSchedulableType(context.getJobDetail().getJobDataMap().getString("type")); + Map params = (Map) context.getJobDetail().getJobDataMap().get("params"); - ///////////// - // run it. // - ///////////// - LOG.debug("Running quartz SQS Poller", logPair("queueName", queueName), logPair("queueProviderName", queueProviderName)); - sqsQueuePoller.run(); + SchedulableRunner schedulableRunner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + schedulableRunner.run(params); } catch(Exception e) { - LOG.warn("Error running SQS Poller", e, logPair("queueName", queueName), logPair("queueProviderName", queueProviderName)); + LOG.warn("Error running QuartzJob", e, logPair("jobContext", context)); } finally { - QContext.clear(); + QContext.init(capturedContext); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java deleted file mode 100644 index 3cfde45f..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzRunProcessJob.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2023. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.scheduler.quartz; - - -import java.io.Serializable; -import java.util.Map; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; -import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; - - -/******************************************************************************* - ** - *******************************************************************************/ -@DisallowConcurrentExecution -public class QuartzRunProcessJob implements Job -{ - private static final QLogger LOG = QLogger.getLogger(QuartzRunProcessJob.class); - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException - { - try - { - JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); - String processName = jobDataMap.getString("processName"); - - /////////////////////////////////////// - // get the process from the instance // - /////////////////////////////////////// - QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); - QProcessMetaData process = qInstance.getProcess(processName); - if(process == null) - { - LOG.warn("Could not find scheduled process in QInstance", logPair("processName", processName)); - return; - } - - /////////////////////////////////////////////// - // if the job has variant data, get it ready // - /////////////////////////////////////////////// - Map backendVariantData = null; - if(jobExecutionContext.getMergedJobDataMap().containsKey("backendVariantData")) - { - backendVariantData = (Map) jobExecutionContext.getMergedJobDataMap().get("backendVariantData"); - } - - ///////////// - // run it. // - ///////////// - LOG.debug("Running quartz process", logPair("processName", processName)); - SchedulerUtils.runProcess(qInstance, QuartzScheduler.getInstance().getSessionSupplier(), qInstance.getProcess(processName), backendVariantData); - } - finally - { - QContext.clear(); - } - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java index 6f682875..f41362c5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzScheduler.java @@ -37,16 +37,15 @@ import java.util.Set; import java.util.TimeZone; import java.util.function.Supplier; import java.util.stream.Collectors; -import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.memoization.AnyKey; import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; import org.quartz.CronExpression; @@ -78,15 +77,10 @@ public class QuartzScheduler implements QSchedulerInterface private final QInstance qInstance; private String schedulerName; - private Properties quartzProperties; private Supplier sessionSupplier; private Scheduler scheduler; - private static final String GROUP_NAME_PROCESSES = "processes"; - private static final String GROUP_NAME_SQS_QUEUES = "sqsQueues"; - private static final String GROUP_NAME_TABLE_AUTOMATIONS = "tableAutomations"; - ///////////////////////////////////////////////////////////////////////////////////////// // create memoization objects for some quartz-query functions, that we'll only want to // @@ -112,17 +106,24 @@ public class QuartzScheduler implements QSchedulerInterface private List scheduledJobsAtStartOfSetup = new ArrayList<>(); private List scheduledJobsAtEndOfSetup = new ArrayList<>(); + ///////////////////////////////////////////////////////////////////////////////// + // track if the instance is past the server's startup routine. // + // for quartz - we'll use this to know if we're allowed to schedule jobs. // + // that is - during server startup, we don't want to the schedule & unschedule // + // routine, which could potentially have serve concurrency problems // + ///////////////////////////////////////////////////////////////////////////////// + private boolean pastStartup = false; + /******************************************************************************* ** Constructor ** *******************************************************************************/ - private QuartzScheduler(QInstance qInstance, String schedulerName, Properties quartzProperties, Supplier sessionSupplier) + private QuartzScheduler(QInstance qInstance, String schedulerName, Supplier sessionSupplier) { this.qInstance = qInstance; this.schedulerName = schedulerName; - this.quartzProperties = quartzProperties; this.sessionSupplier = sessionSupplier; } @@ -136,7 +137,7 @@ public class QuartzScheduler implements QSchedulerInterface { if(quartzScheduler == null) { - quartzScheduler = new QuartzScheduler(qInstance, schedulerName, quartzProperties, sessionSupplier); + quartzScheduler = new QuartzScheduler(qInstance, schedulerName, sessionSupplier); /////////////////////////////////////////////////////////// // Grab the Scheduler instance from the Factory // @@ -165,11 +166,24 @@ public class QuartzScheduler implements QSchedulerInterface + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getSchedulerName() + { + return (schedulerName); + } + + + /******************************************************************************* ** *******************************************************************************/ public void start() { + this.pastStartup = true; + try { ////////////////////// @@ -185,6 +199,17 @@ public class QuartzScheduler implements QSchedulerInterface + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void doNotStart() + { + this.pastStartup = true; + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -225,20 +250,21 @@ public class QuartzScheduler implements QSchedulerInterface ** *******************************************************************************/ @Override - public void setupProcess(QProcessMetaData process, Map backendVariantData, QScheduleMetaData schedule, boolean allowedToStart) + public void setupSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType, Map parameters, QScheduleMetaData schedule, boolean allowedToStart) { - ///////////////////////// - // set up job data map // - ///////////////////////// - Map jobData = new HashMap<>(); - jobData.put("processName", process.getName()); - - if(backendVariantData != null) + //////////////////////////////////////////////////////////////////////////// + // only actually schedule things if we're past the server startup routine // + //////////////////////////////////////////////////////////////////////////// + if(!pastStartup) { - jobData.put("backendVariantData", backendVariantData); + return; } - scheduleJob(process.getName(), GROUP_NAME_PROCESSES, QuartzRunProcessJob.class, jobData, schedule, allowedToStart); + Map jobData = new HashMap<>(); + jobData.put("params", parameters); + jobData.put("type", schedulableType.getName()); + + scheduleJob(schedulableIdentity, schedulableType.getName(), QuartzJobRunner.class, jobData, schedule, allowedToStart); } @@ -249,12 +275,21 @@ public class QuartzScheduler implements QSchedulerInterface @Override public void startOfSetupSchedules() { + //////////////////////////////////////////////////////////////////////////// + // only actually schedule things if we're past the server startup routine // + //////////////////////////////////////////////////////////////////////////// + if(!pastStartup) + { + return; + } + this.insideSetup = true; this.allMemoizations.forEach(m -> m.setTimeout(Duration.ofSeconds(5))); try { this.scheduledJobsAtStartOfSetup = queryQuartz(); + this.scheduledJobsAtEndOfSetup = new ArrayList<>(); } catch(Exception e) { @@ -270,6 +305,14 @@ public class QuartzScheduler implements QSchedulerInterface @Override public void endOfSetupSchedules() { + //////////////////////////////////////////////////////////////////////////// + // only actually schedule things if we're past the server startup routine // + //////////////////////////////////////////////////////////////////////////// + if(!pastStartup) + { + return; + } + this.insideSetup = false; this.allMemoizations.forEach(m -> m.setTimeout(Duration.ofSeconds(0))); @@ -281,7 +324,7 @@ public class QuartzScheduler implements QSchedulerInterface try { Set startJobKeys = this.scheduledJobsAtStartOfSetup.stream().map(w -> w.jobDetail().getKey()).collect(Collectors.toSet()); - Set endJobKeys = scheduledJobsAtEndOfSetup.stream().map(w -> w.jobDetail().getKey()).collect(Collectors.toSet()); + Set endJobKeys = this.scheduledJobsAtEndOfSetup.stream().map(w -> w.jobDetail().getKey()).collect(Collectors.toSet()); ///////////////////////////////////////////////////////////////////////////////////////////////////// // remove all 'end' keys from the set of start keys. any left-over start-keys need to be deleted. // @@ -310,18 +353,19 @@ public class QuartzScheduler implements QSchedulerInterface /******************************************************************************* ** *******************************************************************************/ - private boolean scheduleJob(String jobName, String groupName, Class jobClass, Map jobData, QScheduleMetaData scheduleMetaData, boolean allowedToStart) + private boolean scheduleJob(SchedulableIdentity schedulableIdentity, String groupName, Class jobClass, Map jobData, QScheduleMetaData scheduleMetaData, boolean allowedToStart) { try { ///////////////////////// // Define job instance // ///////////////////////// - JobKey jobKey = new JobKey(jobName, groupName); + JobKey jobKey = new JobKey(schedulableIdentity.getIdentity(), groupName); JobDetail jobDetail = JobBuilder.newJob(jobClass) .withIdentity(jobKey) + .withDescription(schedulableIdentity.getDescription()) .storeDurably() - .requestRecovery() + .requestRecovery() // todo - our frequent repeaters, maybe nice to say false here .build(); jobDetail.getJobDataMap().putAll(jobData); @@ -366,10 +410,11 @@ public class QuartzScheduler implements QSchedulerInterface // Define a Trigger for the schedule // /////////////////////////////////////// Trigger trigger = TriggerBuilder.newTrigger() - .withIdentity(new TriggerKey(jobName, groupName)) + .withIdentity(new TriggerKey(schedulableIdentity.getIdentity(), groupName)) + .withDescription(schedulableIdentity.getDescription() + " - " + getScheduleDescriptionForTrigger(scheduleMetaData)) .forJob(jobKey) .withSchedule(scheduleBuilder) - .startAt(startAt) + // .startAt(startAt) .build(); /////////////////////////////////////// @@ -390,7 +435,7 @@ public class QuartzScheduler implements QSchedulerInterface } catch(Exception e) { - LOG.warn("Error scheduling job", e, logPair("name", jobName), logPair("group", groupName)); + LOG.warn("Error scheduling job", e, logPair("name", schedulableIdentity.getIdentity()), logPair("group", groupName)); return (false); } } @@ -400,17 +445,24 @@ public class QuartzScheduler implements QSchedulerInterface /******************************************************************************* ** *******************************************************************************/ - @Override - public void setupSqsPoller(SQSQueueProviderMetaData queueProvider, QQueueMetaData queue, QScheduleMetaData schedule, boolean allowedToStart) + private String getScheduleDescriptionForTrigger(QScheduleMetaData scheduleMetaData) { - ///////////////////////// - // set up job data map // - ///////////////////////// - Map jobData = new HashMap<>(); - jobData.put("queueProviderName", queueProvider.getName()); - jobData.put("queueName", queue.getName()); + if(StringUtils.hasContent(scheduleMetaData.getDescription())) + { + return scheduleMetaData.getDescription(); + } - scheduleJob(queue.getName(), GROUP_NAME_SQS_QUEUES, QuartzSqsPollerJob.class, jobData, schedule, allowedToStart); + if(StringUtils.hasContent(scheduleMetaData.getCronExpression())) + { + return "cron expression: " + scheduleMetaData.getCronExpression() + (StringUtils.hasContent(scheduleMetaData.getCronTimeZoneId()) ? " time zone: " + scheduleMetaData.getCronTimeZoneId() : ""); + } + + if(scheduleMetaData.getRepeatSeconds() != null) + { + return "repeat seconds: " + scheduleMetaData.getRepeatSeconds(); + } + + return ""; } @@ -419,17 +471,17 @@ public class QuartzScheduler implements QSchedulerInterface ** *******************************************************************************/ @Override - public void setupTableAutomation(QAutomationProviderMetaData automationProvider, PollingAutomationPerTableRunner.TableActionsInterface tableActions, QScheduleMetaData schedule, boolean allowedToStart) + public void unscheduleSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType) { - ///////////////////////// - // set up job data map // - ///////////////////////// - Map jobData = new HashMap<>(); - jobData.put("automationProviderName", automationProvider.getName()); - jobData.put("tableName", tableActions.tableName()); - jobData.put("automationStatus", tableActions.status().toString()); + //////////////////////////////////////////////////////////////////////////// + // only actually schedule things if we're past the server startup routine // + //////////////////////////////////////////////////////////////////////////// + if(!pastStartup) + { + return; + } - scheduleJob(tableActions.tableName() + "." + tableActions.status(), GROUP_NAME_TABLE_AUTOMATIONS, QuartzTableAutomationsJob.class, jobData, schedule, allowedToStart); + deleteJob(new JobKey(schedulableIdentity.getIdentity(), schedulableType.getName())); } @@ -438,9 +490,19 @@ public class QuartzScheduler implements QSchedulerInterface ** *******************************************************************************/ @Override - public void unscheduleProcess(QProcessMetaData process) + public void unscheduleAll() throws QException { - deleteJob(new JobKey(process.getName(), GROUP_NAME_PROCESSES)); + try + { + for(QuartzJobAndTriggerWrapper wrapper : queryQuartz()) + { + deleteJob(new JobKey(wrapper.jobDetail().getKey().getName(), wrapper.jobDetail().getKey().getGroup())); + } + } + catch(Exception e) + { + throw (new QException("Error unscheduling all quartz jobs", e)); + } } @@ -455,9 +517,9 @@ public class QuartzScheduler implements QSchedulerInterface { boolean wasPaused = wasExistingJobPaused(jobKey); - this.scheduler.addJob(jobDetail, true); // note, true flag here replaces if already present. - this.scheduler.rescheduleJob(trigger.getKey(), trigger); + this.scheduler.scheduleJob(jobDetail, Set.of(trigger), true); // note, true flag here replaces if already present. LOG.info("Re-scheduled job", logPair("jobKey", jobKey)); + if(wasPaused) { LOG.info("Re-pausing job", logPair("jobKey", jobKey)); @@ -488,7 +550,7 @@ public class QuartzScheduler implements QSchedulerInterface } } - return(false); + return (false); } @@ -583,7 +645,20 @@ public class QuartzScheduler implements QSchedulerInterface *******************************************************************************/ public void pauseAll() throws SchedulerException { - this.scheduler.pauseAll(); + /////////////////////////////////////////////////////////////////////////////// + // lesson from past self to future self: // + // pauseAll creates paused-group entries for all jobs - // + // and so they can only really be resumed by a resumeAll call... // + // even newly scheduled things become paused. Which can be quite confusing. // + // so, we don't want pause all. // + /////////////////////////////////////////////////////////////////////////////// + // this.scheduler.pauseAll(); + + List quartzJobAndTriggerWrappers = queryQuartz(); + for(QuartzJobAndTriggerWrapper wrapper : quartzJobAndTriggerWrappers) + { + this.pauseJob(wrapper.jobDetail().getKey().getName(), wrapper.jobDetail().getKey().getGroup()); + } } @@ -593,6 +668,9 @@ public class QuartzScheduler implements QSchedulerInterface *******************************************************************************/ public void resumeAll() throws SchedulerException { + ////////////////////////////////////////////////// + // this seems okay, even though pauseAll isn't. // + ////////////////////////////////////////////////// this.scheduler.resumeAll(); } @@ -603,6 +681,7 @@ public class QuartzScheduler implements QSchedulerInterface *******************************************************************************/ public void pauseJob(String jobName, String groupName) throws SchedulerException { + LOG.info("Request to pause job", logPair("jobName", jobName)); this.scheduler.pauseJob(new JobKey(jobName, groupName)); } @@ -613,6 +692,7 @@ public class QuartzScheduler implements QSchedulerInterface *******************************************************************************/ public void resumeJob(String jobName, String groupName) throws SchedulerException { + LOG.info("Request to resume job", logPair("jobName", jobName)); this.scheduler.resumeJob(new JobKey(jobName, groupName)); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java deleted file mode 100644 index c88fa495..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTableAutomationsJob.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.scheduler.quartz; - - -import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; -import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; -import org.quartz.DisallowConcurrentExecution; -import org.quartz.Job; -import org.quartz.JobDataMap; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; -import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; - - -/******************************************************************************* - ** - *******************************************************************************/ -@DisallowConcurrentExecution -public class QuartzTableAutomationsJob implements Job -{ - private static final QLogger LOG = QLogger.getLogger(QuartzTableAutomationsJob.class); - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException - { - String tableName = null; - String automationProviderName = null; - AutomationStatus automationStatus = null; - - try - { - JobDataMap jobDataMap = jobExecutionContext.getJobDetail().getJobDataMap(); - tableName = jobDataMap.getString("tableName"); - automationProviderName = jobDataMap.getString("automationProviderName"); - automationStatus = AutomationStatus.valueOf(jobDataMap.getString("automationStatus")); - QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); - - QTableMetaData table = qInstance.getTable(tableName); - if(table == null) - { - LOG.warn("Could not find table for automations in QInstance", logPair("tableName", tableName)); - return; - } - - QTableAutomationDetails automationDetails = table.getAutomationDetails(); - if(automationDetails == null) - { - LOG.warn("Could not find automationDetails for table for automations in QInstance", logPair("tableName", tableName)); - return; - } - - /////////////////////////////////// - // todo - sharded automations... // - /////////////////////////////////// - PollingAutomationPerTableRunner.TableActionsInterface tableAction = new PollingAutomationPerTableRunner.TableActions(tableName, automationDetails, automationStatus); - PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProviderName, QuartzScheduler.getInstance().getSessionSupplier(), tableAction); - - ///////////// - // run it. // - ///////////// - LOG.debug("Running Table Automations", logPair("tableName", tableName), logPair("automationStatus", automationStatus)); - runner.run(); - } - catch(Exception e) - { - LOG.warn("Error running Table Automations", e, logPair("tableName", tableName), logPair("automationStatus", automationStatus)); - } - finally - { - QContext.clear(); - } - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java index 7d820f08..123b48c9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseAllQuartzJobsProcess.java @@ -40,7 +40,7 @@ import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; /******************************************************************************* - ** + ** Manage process to pause all quartz jobs *******************************************************************************/ public class PauseAllQuartzJobsProcess implements BackendStep, MetaDataProducerInterface { @@ -55,6 +55,10 @@ public class PauseAllQuartzJobsProcess implements BackendStep, MetaDataProducerI .withName(getClass().getSimpleName()) .withLabel("Pause All Quartz Jobs") .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to pause all quartz jobs."))), new QBackendStepMetaData() .withName("execute") .withCode(new QCodeReference(getClass())), diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java index 9cd769c7..7d81e31f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeAllQuartzJobsProcess.java @@ -40,7 +40,7 @@ import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; /******************************************************************************* - ** + ** Manage process to resume all quartz jobs *******************************************************************************/ public class ResumeAllQuartzJobsProcess implements BackendStep, MetaDataProducerInterface { @@ -55,6 +55,10 @@ public class ResumeAllQuartzJobsProcess implements BackendStep, MetaDataProducer .withName(getClass().getSimpleName()) .withLabel("Resume All Quartz Jobs") .withStepList(List.of( + new QFrontendStepMetaData() + .withName("confirm") + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine().withVelocityTemplate("Please confirm you wish to resume all quartz jobs."))), new QBackendStepMetaData() .withName("execute") .withCode(new QCodeReference(getClass())), diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/SchedulableType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/SchedulableType.java new file mode 100644 index 00000000..8f4d04c1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/SchedulableType.java @@ -0,0 +1,98 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable; + + +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SchedulableType +{ + private String name; + private QCodeReference runner; + + + + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + public String getName() + { + return (this.name); + } + + + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + public SchedulableType withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for runner + *******************************************************************************/ + public QCodeReference getRunner() + { + return (this.runner); + } + + + + /******************************************************************************* + ** Setter for runner + *******************************************************************************/ + public void setRunner(QCodeReference runner) + { + this.runner = runner; + } + + + + /******************************************************************************* + ** Fluent setter for runner + *******************************************************************************/ + public SchedulableType withRunner(QCodeReference runner) + { + this.runner = runner; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/BasicSchedulableIdentity.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/BasicSchedulableIdentity.java new file mode 100644 index 00000000..2ad2a14d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/BasicSchedulableIdentity.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.identity; + + +import java.util.Objects; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** Basic implementation of interface for identifying schedulable things + *******************************************************************************/ +public class BasicSchedulableIdentity implements SchedulableIdentity +{ + private String identity; + private String description; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BasicSchedulableIdentity(String identity, String description) + { + if(!StringUtils.hasContent(identity)) + { + throw (new IllegalArgumentException("Identity may not be null or empty.")); + } + + this.identity = identity; + this.description = description; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean equals(Object o) + { + if(this == o) + { + return true; + } + + if(o == null || getClass() != o.getClass()) + { + return false; + } + + BasicSchedulableIdentity that = (BasicSchedulableIdentity) o; + return Objects.equals(identity, that.identity); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int hashCode() + { + return Objects.hash(identity); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getIdentity() + { + return identity; + } + + + + /******************************************************************************* + ** Getter for description + ** + *******************************************************************************/ + @Override + public String getDescription() + { + return description; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return getIdentity(); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentity.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentity.java new file mode 100644 index 00000000..f4cb1334 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentity.java @@ -0,0 +1,55 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.identity; + + +/******************************************************************************* + ** Unique identifier for a thing that can be scheduled + *******************************************************************************/ +public interface SchedulableIdentity +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Override + boolean equals(Object that); + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + int hashCode(); + + + /******************************************************************************* + ** + *******************************************************************************/ + String getIdentity(); + + + /******************************************************************************* + ** should NOT be part of equals & has code + *******************************************************************************/ + String getDescription(); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java new file mode 100644 index 00000000..01b1d17b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java @@ -0,0 +1,96 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.identity; + + +import java.util.HashMap; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; + + +/******************************************************************************* + ** Factory to produce SchedulableIdentity objects + *******************************************************************************/ +public class SchedulableIdentityFactory +{ + + /******************************************************************************* + ** Factory to create one of these for a scheduled job record + *******************************************************************************/ + public static BasicSchedulableIdentity of(ScheduledJob scheduledJob) + { + String description = ""; + ScheduledJobType scheduledJobType = ScheduledJobType.getById(scheduledJob.getType()); + if(scheduledJobType != null) + { + try + { + SchedulableType schedulableType = QContext.getQInstance().getSchedulableType(scheduledJob.getType()); + SchedulableRunner runner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + description = runner.getDescription(new HashMap<>(scheduledJob.getJobParametersMap())); + } + catch(Exception e) + { + description = "type: " + scheduledJobType; + } + } + + return new BasicSchedulableIdentity("scheduledJob:" + scheduledJob.getId(), description); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static BasicSchedulableIdentity of(QProcessMetaData process) + { + return new BasicSchedulableIdentity("process:" + process.getName(), "Process: " + process.getName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static SchedulableIdentity of(QQueueMetaData queue) + { + return new BasicSchedulableIdentity("queue:" + queue.getName(), "Queue: " + queue.getName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static SchedulableIdentity of(PollingAutomationPerTableRunner.TableActionsInterface tableActions) + { + return new BasicSchedulableIdentity("tableAutomations:" + tableActions.tableName() + "." + tableActions.status(), "TableAutomations: " + tableActions.tableName() + "." + tableActions.status()); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableProcessRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableProcessRunner.java new file mode 100644 index 00000000..c94d25ba --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableProcessRunner.java @@ -0,0 +1,188 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.runner; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Schedulable process runner - e.g., how a QProcess is run by a scheduler. + *******************************************************************************/ +public class SchedulableProcessRunner implements SchedulableRunner +{ + private static final QLogger LOG = QLogger.getLogger(SchedulableProcessRunner.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(Map params) + { + String processName = ValueUtils.getValueAsString(params.get("processName")); + + /////////////////////////////////////// + // get the process from the instance // + /////////////////////////////////////// + QInstance qInstance = QContext.getQInstance(); + QProcessMetaData process = qInstance.getProcess(processName); + if(process == null) + { + LOG.warn("Could not find scheduled process in QInstance", logPair("processName", processName)); + return; + } + + /////////////////////////////////////////////// + // if the job has variant data, get it ready // + /////////////////////////////////////////////// + Map backendVariantData = null; + if(params.containsKey("backendVariantData")) + { + backendVariantData = (Map) params.get("backendVariantData"); + } + + Map processInputValues = buildProcessInputValuesMap(params, process); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running scheduled process", logPair("processName", processName)); + SchedulerUtils.runProcess(qInstance, () -> QContext.getQSession(), qInstance.getProcess(processName), backendVariantData, processInputValues); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void validateParams(SchedulableIdentity schedulableIdentity, Map paramMap) throws QException + { + String processName = ValueUtils.getValueAsString(paramMap.get("processName")); + if(!StringUtils.hasContent(processName)) + { + throw (new QException("Missing scheduledJobParameter with key [processName] in " + schedulableIdentity)); + } + + QProcessMetaData process = QContext.getQInstance().getProcess(processName); + if(process == null) + { + throw (new QException("Unrecognized processName [" + processName + "] in " + schedulableIdentity)); + } + + if(process.getSchedule() != null) + { + throw (new QException("Process [" + processName + "] has a schedule in its metaData - so it should not be dynamically scheduled via a scheduled job! " + schedulableIdentity)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDescription(Map params) + { + return "Process: " + params.get("processName"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static Map buildProcessInputValuesMap(Map params, QProcessMetaData process) + { + Map processInputValues = new HashMap<>(); + + ////////////////////////////////////////////////////////////////////////////////////// + // track which keys need processed - start by removing ones we know we handle above // + ////////////////////////////////////////////////////////////////////////////////////// + Set keys = new HashSet<>(params.keySet()); + keys.remove("processName"); + keys.remove("backendVariantData"); + + if(!keys.isEmpty()) + { + ////////////////////////////////////////////////////////////////////////// + // first make a pass going over the process's identified input fields - // + // getting values from the quartz job data map, and putting them into // + // the process input value map as the field's type (if we can) // + ////////////////////////////////////////////////////////////////////////// + for(QFieldMetaData inputField : process.getInputFields()) + { + String fieldName = inputField.getName(); + if(params.containsKey(fieldName)) + { + Object value = params.get(fieldName); + try + { + processInputValues.put(fieldName, ValueUtils.getValueAsFieldType(inputField.getType(), value)); + keys.remove(fieldName); + } + catch(Exception e) + { + LOG.warn("Error getting process input value from quartz job data map", e, logPair("fieldName", fieldName), logPair("value", value)); + } + } + } + + //////////////////////////////////////////////////////////////////////////////////////// + // if any values are left in the map (based on keys set that we're removing from) // + // then try to put those in the input map (assuming they can be cast to Serializable) // + //////////////////////////////////////////////////////////////////////////////////////// + for(String key : keys) + { + Object value = params.get(key); + try + { + processInputValues.put(key, (Serializable) value); + } + catch(Exception e) + { + LOG.warn("Error getting process input value from quartz job data map", e, logPair("key", key), logPair("value", value)); + } + } + } + + return processInputValues; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableRunner.java new file mode 100644 index 00000000..b718d609 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableRunner.java @@ -0,0 +1,51 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.runner; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; + + +/******************************************************************************* + ** Interface for different types of schedulabe things that can be run + *******************************************************************************/ +public interface SchedulableRunner +{ + + /******************************************************************************* + ** + *******************************************************************************/ + void run(Map params); + + + /******************************************************************************* + ** + *******************************************************************************/ + void validateParams(SchedulableIdentity schedulableIdentity, Map paramMap) throws QException; + + /******************************************************************************* + ** + *******************************************************************************/ + String getDescription(Map params); +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableSQSQueueRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableSQSQueueRunner.java new file mode 100644 index 00000000..f2cc4dab --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableSQSQueueRunner.java @@ -0,0 +1,135 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.runner; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Schedulable SQSQueue runner - e.g., how an SQSQueuePoller is run by a scheduler. + *******************************************************************************/ +public class SchedulableSQSQueueRunner implements SchedulableRunner +{ + private static final QLogger LOG = QLogger.getLogger(SchedulableSQSQueueRunner.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(Map params) + { + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + + String queueName = ValueUtils.getValueAsString(params.get("queueName")); + if(!StringUtils.hasContent(queueName)) + { + LOG.warn("Missing queueName in params."); + return; + } + + QQueueMetaData queue = qInstance.getQueue(queueName); + if(queue == null) + { + LOG.warn("Unrecognized queueName [" + queueName + "]"); + return; + } + + QQueueProviderMetaData queueProvider = qInstance.getQueueProvider(queue.getProviderName()); + if(!(queueProvider instanceof SQSQueueProviderMetaData)) + { + LOG.warn("Queue [" + queueName + "] is of an unsupported queue provider type (not SQS)"); + return; + } + + SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); + sqsQueuePoller.setQueueMetaData(queue); + sqsQueuePoller.setQueueProviderMetaData((SQSQueueProviderMetaData) queueProvider); + sqsQueuePoller.setQInstance(qInstance); + sqsQueuePoller.setSessionSupplier(QuartzScheduler.getInstance().getSessionSupplier()); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running SQS Queue poller", logPair("queueName", queueName)); + sqsQueuePoller.run(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void validateParams(SchedulableIdentity schedulableIdentity, Map paramMap) throws QException + { + String queueName = ValueUtils.getValueAsString(paramMap.get("queueName")); + if(!StringUtils.hasContent(queueName)) + { + throw (new QException("Missing scheduledJobParameter with key [queueName] in " + schedulableIdentity)); + } + + QQueueMetaData queue = QContext.getQInstance().getQueue(queueName); + if(queue == null) + { + throw (new QException("Unrecognized queueName [" + queueName + "] in " + schedulableIdentity)); + } + + QQueueProviderMetaData queueProvider = QContext.getQInstance().getQueueProvider(queue.getProviderName()); + if(!(queueProvider instanceof SQSQueueProviderMetaData)) + { + throw (new QException("Queue [" + queueName + "] is of an unsupported queue provider type (not SQS) in " + schedulableIdentity)); + } + + if(queue.getSchedule() != null) + { + throw (new QException("Queue [" + queueName + "] has a schedule in its metaData - so it should not be dynamically scheduled via a scheduled job! " + schedulableIdentity)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDescription(Map params) + { + return "Queue: " + params.get("queueName"); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java new file mode 100644 index 00000000..5a6e0e53 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java @@ -0,0 +1,168 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.schedulable.runner; + + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Schedulable TableAutomations runner - e.g., how a table automations are run + ** by a scheduler. + *******************************************************************************/ +public class SchedulableTableAutomationsRunner implements SchedulableRunner +{ + private static final QLogger LOG = QLogger.getLogger(SchedulableTableAutomationsRunner.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(Map params) + { + QInstance qInstance = QuartzScheduler.getInstance().getQInstance(); + + String tableName = ValueUtils.getValueAsString(params.get("tableName")); + if(!StringUtils.hasContent(tableName)) + { + LOG.warn("Missing tableName in params."); + return; + } + + QTableMetaData table = qInstance.getTable(tableName); + if(table == null) + { + LOG.warn("Unrecognized tableName [" + tableName + "]"); + return; + } + + AutomationStatus automationStatus = AutomationStatus.valueOf(ValueUtils.getValueAsString(params.get("automationStatus"))); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running table automations", logPair("tableName", tableName), logPair("")); + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + if(automationDetails == null) + { + LOG.warn("Could not find automationDetails for table for automations in QInstance", logPair("tableName", tableName)); + return; + } + + /////////////////////////////////// + // todo - sharded automations... // + /////////////////////////////////// + PollingAutomationPerTableRunner.TableActionsInterface tableAction = new PollingAutomationPerTableRunner.TableActions(tableName, automationDetails, automationStatus); + PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationDetails.getProviderName(), QuartzScheduler.getInstance().getSessionSupplier(), tableAction); + + ///////////// + // run it. // + ///////////// + LOG.debug("Running Table Automations", logPair("tableName", tableName), logPair("automationStatus", automationStatus)); + runner.run(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void validateParams(SchedulableIdentity schedulableIdentity, Map paramMap) throws QException + { + String tableName = ValueUtils.getValueAsString(paramMap.get("tableName")); + if(!StringUtils.hasContent(tableName)) + { + throw (new QException("Missing scheduledJobParameter with key [tableName] in " + schedulableIdentity)); + } + + String automationStatus = ValueUtils.getValueAsString(paramMap.get("automationStatus")); + if(!StringUtils.hasContent(automationStatus)) + { + throw (new QException("Missing scheduledJobParameter with key [automationStatus] in " + schedulableIdentity)); + } + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + if(table == null) + { + throw (new QException("Unrecognized tableName [" + tableName + "] in " + schedulableIdentity)); + } + + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + if(automationDetails == null) + { + throw (new QException("Table [" + tableName + "] does not have automationDetails in " + schedulableIdentity)); + } + + if(automationDetails.getSchedule() != null) + { + throw (new QException("Table [" + tableName + "] automationDetails has a schedule in its metaData - so it should not be dynamically scheduled via a scheduled job! " + schedulableIdentity)); + } + + QAutomationProviderMetaData automationProvider = QContext.getQInstance().getAutomationProvider(automationDetails.getProviderName()); + + List tableActionList = PollingAutomationPerTableRunner.getTableActions(QContext.getQInstance(), automationProvider.getName()); + for(PollingAutomationPerTableRunner.TableActionsInterface tableActions : tableActionList) + { + if(tableActions.status().name().equals(automationStatus)) + { + return; + } + } + + ///////////////////////////////////////////////////////////////////////////////////// + // if we get out of the loop, it means we didn't find a matching status - so throw // + ///////////////////////////////////////////////////////////////////////////////////// + throw (new QException("Did not find table automation actions matching automationStatus [" + automationStatus + "] for table [" + tableName + "] in " + schedulableIdentity + + " (Found: " + tableActionList.stream().map(ta -> ta.status().name()).collect(Collectors.joining(",")) + ")")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDescription(Map params) + { + return "TableAutomations: " + params.get("tableName") + "." + params.get("automationStatus"); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java new file mode 100644 index 00000000..29e6e976 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java @@ -0,0 +1,87 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler.simple; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.context.CapturedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SimpleJobRunner implements Runnable +{ + private static final QLogger LOG = QLogger.getLogger(SimpleJobRunner.class); + + private QInstance qInstance; + private SchedulableType schedulableType; + private Map params; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SimpleJobRunner(QInstance qInstance, SchedulableType type, Map params) + { + this.qInstance = qInstance; + this.schedulableType = type; + this.params = params; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run() + { + CapturedContext capturedContext = QContext.capture(); + try + { + SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); + QContext.init(qInstance, simpleScheduler.getSessionSupplier().get()); + + SchedulableRunner schedulableRunner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + schedulableRunner.run(params); + } + catch(Exception e) + { + LOG.warn("Error running SimpleScheduler job", e, logPair("params", params)); + } + finally + { + QContext.init(capturedContext); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java index ac7ab640..04ff9a6f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleScheduler.java @@ -24,22 +24,21 @@ package com.kingsrook.qqq.backend.core.scheduler.simple; import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.Supplier; -import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner; -import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.scheduler.QSchedulerInterface; -import com.kingsrook.qqq.backend.core.scheduler.SchedulerUtils; -import org.apache.commons.lang.NotImplementedException; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.SchedulableIdentity; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -67,7 +66,7 @@ public class SimpleScheduler implements QSchedulerInterface ///////////////////////////////////////////////////////////////////////////////////// private int delayIndex = 0; - private List executors = new ArrayList<>(); + private Map executors = new LinkedHashMap<>(); @@ -101,7 +100,7 @@ public class SimpleScheduler implements QSchedulerInterface @Override public void start() { - for(StandardScheduledExecutor executor : executors) + for(StandardScheduledExecutor executor : executors.values()) { executor.start(); } @@ -115,7 +114,7 @@ public class SimpleScheduler implements QSchedulerInterface @Override public void stopAsync() { - for(StandardScheduledExecutor scheduledExecutor : executors) + for(StandardScheduledExecutor scheduledExecutor : executors.values()) { scheduledExecutor.stopAsync(); } @@ -129,7 +128,7 @@ public class SimpleScheduler implements QSchedulerInterface @Override public void stop() { - for(StandardScheduledExecutor scheduledExecutor : executors) + for(StandardScheduledExecutor scheduledExecutor : executors.values()) { scheduledExecutor.stop(); } @@ -141,83 +140,18 @@ public class SimpleScheduler implements QSchedulerInterface ** *******************************************************************************/ @Override - public void setupTableAutomation(QAutomationProviderMetaData automationProvider, PollingAutomationPerTableRunner.TableActionsInterface tableActions, QScheduleMetaData schedule, boolean allowedToStart) + public void setupSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType, Map parameters, QScheduleMetaData schedule, boolean allowedToStart) { if(!allowedToStart) { return; } - PollingAutomationPerTableRunner runner = new PollingAutomationPerTableRunner(qInstance, automationProvider.getName(), sessionSupplier, tableActions); - StandardScheduledExecutor executor = new StandardScheduledExecutor(runner); - - executor.setName(runner.getName()); + SimpleJobRunner simpleJobRunner = new SimpleJobRunner(qInstance, schedulableType, new HashMap<>(parameters)); + StandardScheduledExecutor executor = new StandardScheduledExecutor(simpleJobRunner); + executor.setName(schedulableIdentity.getIdentity()); setScheduleInExecutor(schedule, executor); - executors.add(executor); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void unscheduleProcess(QProcessMetaData process) - { - throw (new NotImplementedException("Unscheduling is not implemented in SimpleScheduler...")); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void setupSqsPoller(SQSQueueProviderMetaData queueProvider, QQueueMetaData queue, QScheduleMetaData schedule, boolean allowedToStart) - { - if(!allowedToStart) - { - return; - } - - QInstance scheduleManagerQueueInstance = qInstance; - Supplier scheduleManagerSessionSupplier = sessionSupplier; - - SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); - sqsQueuePoller.setQueueProviderMetaData(queueProvider); - sqsQueuePoller.setQueueMetaData(queue); - sqsQueuePoller.setQInstance(scheduleManagerQueueInstance); - sqsQueuePoller.setSessionSupplier(scheduleManagerSessionSupplier); - - StandardScheduledExecutor executor = new StandardScheduledExecutor(sqsQueuePoller); - - executor.setName(queue.getName()); - setScheduleInExecutor(schedule, executor); - executors.add(executor); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void setupProcess(QProcessMetaData process, Map backendVariantData, QScheduleMetaData schedule, boolean allowedToStart) - { - if(!allowedToStart) - { - return; - } - - Runnable runProcess = () -> - { - SchedulerUtils.runProcess(qInstance, sessionSupplier, process, backendVariantData); - }; - - StandardScheduledExecutor executor = new StandardScheduledExecutor(runProcess); - executor.setName("process:" + process.getName()); - setScheduleInExecutor(schedule, executor); - executors.add(executor); + executors.put(schedulableIdentity, executor); } @@ -255,13 +189,34 @@ public class SimpleScheduler implements QSchedulerInterface /******************************************************************************* ** *******************************************************************************/ - private QScheduleMetaData getDefaultSchedule() + @Override + public void unscheduleSchedulable(SchedulableIdentity schedulableIdentity, SchedulableType schedulableType) { - QScheduleMetaData schedule; - schedule = new QScheduleMetaData() - .withInitialDelaySeconds(delayIndex++) - .withRepeatSeconds(60); - return schedule; + StandardScheduledExecutor executor = executors.get(schedulableIdentity); + if(executor != null) + { + LOG.info("Stopping job in simple scheduler", logPair("identity", schedulableIdentity)); + executors.remove(schedulableIdentity); + executor.stop(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void unscheduleAll() throws QException + { + for(Map.Entry entry : new HashSet<>(executors.entrySet())) + { + StandardScheduledExecutor executor = executors.remove(entry.getKey()); + if(executor != null) + { + executor.stopAsync(); + } + } } @@ -278,21 +233,23 @@ public class SimpleScheduler implements QSchedulerInterface /******************************************************************************* - ** Getter for managedExecutors + ** Getter for sessionSupplier ** *******************************************************************************/ - public List getExecutors() + public Supplier getSessionSupplier() { - return executors; + return sessionSupplier; } /******************************************************************************* + ** Getter for managedExecutors ** *******************************************************************************/ - static void resetSingleton() + public List getExecutors() { + return new ArrayList<>(executors.values()); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java new file mode 100644 index 00000000..d5e97c60 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java @@ -0,0 +1,205 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.util.ArrayList; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobParameter; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Unit test for QScheduleManager + *******************************************************************************/ +class QScheduleManagerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QLogger.deactivateCollectingLoggerForClass(QuartzScheduler.class); + + try + { + QScheduleManager.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + + try + { + QuartzScheduler.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private ScheduledJob newScheduledJob(ScheduledJobType type, Map params) + { + ScheduledJob scheduledJob = new ScheduledJob() + .withId(1) + .withIsActive(true) + .withSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME) + .withType(type.getId()) + .withRepeatSeconds(1) + .withJobParameters(new ArrayList<>()); + + for(Map.Entry entry : params.entrySet()) + { + scheduledJob.getJobParameters().add(new ScheduledJobParameter().withKey(entry.getKey()).withValue(entry.getValue())); + } + + return (scheduledJob); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSetupScheduledJobErrorCases() throws QException + { + QScheduleManager qScheduleManager = QScheduleManager.initInstance(QContext.getQInstance(), () -> QContext.getQSession()); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, Map.of()).withRepeatSeconds(null))) + .hasMessageContaining("Missing a schedule"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, Map.of()).withType(null))) + .hasMessageContaining("Missing a type"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, Map.of()).withType("notAType"))) + .hasMessageContaining("Unrecognized type"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, Map.of()))) + .hasMessageContaining("Missing scheduledJobParameter with key [processName]"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, Map.of("processName", "notAProcess")))) + .hasMessageContaining("Unrecognized processName"); + + QContext.getQInstance().getProcess(TestUtils.PROCESS_NAME_BASEPULL).withSchedule(new QScheduleMetaData()); + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, Map.of("processName", TestUtils.PROCESS_NAME_BASEPULL)))) + .hasMessageContaining("has a schedule in its metaData - so it should not be dynamically scheduled"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.QUEUE_PROCESSOR, Map.of()))) + .hasMessageContaining("Missing scheduledJobParameter with key [queueName]"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.QUEUE_PROCESSOR, Map.of("queueName", "notAQueue")))) + .hasMessageContaining("Unrecognized queueName"); + + QContext.getQInstance().getQueue(TestUtils.TEST_SQS_QUEUE).withSchedule(new QScheduleMetaData()); + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.QUEUE_PROCESSOR, Map.of("queueName", TestUtils.TEST_SQS_QUEUE)))) + .hasMessageContaining("has a schedule in its metaData - so it should not be dynamically scheduled"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of()))) + .hasMessageContaining("Missing scheduledJobParameter with key [tableName]"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", "notATable")))) + .hasMessageContaining("Missing scheduledJobParameter with key [automationStatus]"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", "notATable", "automationStatus", AutomationStatus.PENDING_INSERT_AUTOMATIONS.name())))) + .hasMessageContaining("Unrecognized tableName"); + + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC, "automationStatus", AutomationStatus.PENDING_INSERT_AUTOMATIONS.name())))) + .hasMessageContaining("does not have automationDetails"); + + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().withSchedule(null); + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY, "automationStatus", "foobar")))) + .hasMessageContaining("Did not find table automation actions matching automationStatus") + .hasMessageContaining("Found: PENDING_INSERT_AUTOMATIONS,PENDING_UPDATE_AUTOMATIONS"); + + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().withSchedule(new QScheduleMetaData()); + assertThatThrownBy(() -> qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY, "automationStatus", AutomationStatus.PENDING_INSERT_AUTOMATIONS.name())))) + .hasMessageContaining("has a schedule in its metaData - so it should not be dynamically scheduled"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessfulScheduleWithQuartz() throws QException + { + QCollectingLogger quartzLogger = QLogger.activateCollectingLoggerForClass(QuartzScheduler.class); + + QInstance qInstance = QContext.getQInstance(); + QuartzTestUtils.setupInstanceForQuartzTests(); + + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> QContext.getQSession()); + qScheduleManager.start(); + + qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.PROCESS, + Map.of("processName", TestUtils.PROCESS_NAME_GREET_PEOPLE)) + .withId(2) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + + qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.QUEUE_PROCESSOR, + Map.of("queueName", TestUtils.TEST_SQS_QUEUE)) + .withId(3) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + + qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, + Map.of("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY, "automationStatus", AutomationStatus.PENDING_UPDATE_AUTOMATIONS.name())) + .withId(4) + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + + assertThat(quartzLogger.getCollectedMessages()) + .anyMatch(l -> l.getMessage().matches(".*Scheduled new job.*PROCESS.scheduledJob:2.*")) + .anyMatch(l -> l.getMessage().matches(".*Scheduled new job.*QUEUE_PROCESSOR.scheduledJob:3.*")) + .anyMatch(l -> l.getMessage().matches(".*Scheduled new job.*TABLE_AUTOMATIONS.scheduledJob:4.*")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java new file mode 100644 index 00000000..16b1a427 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerTestUtils.java @@ -0,0 +1,79 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SchedulerTestUtils +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessMetaData buildTestProcess(String name, String schedulerName) + { + return new QProcessMetaData() + .withName(name) + .withSchedule(new QScheduleMetaData() + .withSchedulerName(schedulerName) + .withRepeatMillis(2) + .withInitialDelaySeconds(0)) + .withStepList(List.of(new QBackendStepMetaData() + .withName("step") + .withCode(new QCodeReference(BasicStep.class)))); + } + + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class BasicStep implements BackendStep + { + public static int counter = 0; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + counter++; + } + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java index d3b3ba66..43fe0b1d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java @@ -22,23 +22,24 @@ package com.kingsrook.qqq.backend.core.scheduler.quartz; +import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils.BasicStep; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.identity.BasicSchedulableIdentity; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableSQSQueueRunner; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableTableAutomationsRunner; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import org.apache.logging.log4j.Level; import org.junit.jupiter.api.AfterEach; @@ -100,20 +101,21 @@ class QuartzSchedulerTest extends BaseTest ////////////////////////////////////////////////////////////////////////////////////////////////////// // set these runners to use collecting logger, so we can assert that they did run, and didn't throw // ////////////////////////////////////////////////////////////////////////////////////////////////////// - QCollectingLogger quartzSqsPollerJobLog = QLogger.activateCollectingLoggerForClass(QuartzSqsPollerJob.class); - QCollectingLogger quartzTableAutomationsJobLog = QLogger.activateCollectingLoggerForClass(QuartzTableAutomationsJob.class); + QCollectingLogger quartzSqsPollerJobLog = QLogger.activateCollectingLoggerForClass(SchedulableSQSQueueRunner.class); + QCollectingLogger quartzTableAutomationsJobLog = QLogger.activateCollectingLoggerForClass(SchedulableTableAutomationsRunner.class); ////////////////////////////////////////// // add a process we can run and observe // ////////////////////////////////////////// - qInstance.addProcess(buildTestProcess("testScheduledProcess")); + qInstance.addProcess(SchedulerTestUtils.buildTestProcess("testScheduledProcess", QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); - ////////////////////////////////////////////////////////////////////////////// - // start the schedule manager, which will schedule things, and start quartz // - ////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////// + // start the schedule manager, then ask it to set up all schedules // + ///////////////////////////////////////////////////////////////////// QSession qSession = QContext.getQSession(); QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); qScheduleManager.start(); + qScheduleManager.setupAllSchedules(); ////////////////////////////////////////////////// // give a moment for the job to run a few times // @@ -128,7 +130,7 @@ class QuartzSchedulerTest extends BaseTest // make sure poller ran, and didn't issue any warns // ////////////////////////////////////////////////////// assertThat(quartzSqsPollerJobLog.getCollectedMessages()) - .anyMatch(m -> m.getLevel().equals(Level.DEBUG) && m.getMessage().contains("Running quartz SQS Poller")) + .anyMatch(m -> m.getLevel().equals(Level.DEBUG) && m.getMessage().contains("Running SQS Queue poller")) .noneMatch(m -> m.getLevel().equals(Level.WARN)); ////////////////////////////////////////////////////// @@ -140,30 +142,13 @@ class QuartzSchedulerTest extends BaseTest } finally { - QLogger.deactivateCollectingLoggerForClass(QuartzSqsPollerJob.class); + QLogger.deactivateCollectingLoggerForClass(SchedulableSQSQueueRunner.class); + QLogger.deactivateCollectingLoggerForClass(SchedulableTableAutomationsRunner.class); } } - /******************************************************************************* - ** - *******************************************************************************/ - private static QProcessMetaData buildTestProcess(String name) - { - return new QProcessMetaData() - .withName(name) - .withSchedule(new QScheduleMetaData() - .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) - .withRepeatMillis(2) - .withInitialDelaySeconds(0)) - .withStepList(List.of(new QBackendStepMetaData() - .withName("step") - .withCode(new QCodeReference(BasicStep.class)))); - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -171,45 +156,33 @@ class QuartzSchedulerTest extends BaseTest void testRemovingNoLongerNeededJobsDuringSetupSchedules() throws SchedulerException { QInstance qInstance = QContext.getQInstance(); + QScheduleManager.defineDefaultSchedulableTypesInInstance(qInstance); QuartzTestUtils.setupInstanceForQuartzTests(); //////////////////////////// // put two jobs in quartz // //////////////////////////// - QProcessMetaData test1 = buildTestProcess("test1"); - QProcessMetaData test2 = buildTestProcess("test2"); + QProcessMetaData test1 = SchedulerTestUtils.buildTestProcess("test1", QuartzTestUtils.QUARTZ_SCHEDULER_NAME); + QProcessMetaData test2 = SchedulerTestUtils.buildTestProcess("test2", QuartzTestUtils.QUARTZ_SCHEDULER_NAME); qInstance.addProcess(test1); qInstance.addProcess(test2); + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.PROCESS.getId()); + QuartzScheduler quartzScheduler = QuartzScheduler.initInstance(qInstance, QuartzTestUtils.QUARTZ_SCHEDULER_NAME, QuartzTestUtils.getQuartzProperties(), () -> QContext.getQSession()); - quartzScheduler.setupProcess(test1, null, test1.getSchedule(), false); - quartzScheduler.setupProcess(test2, null, test2.getSchedule(), false); + quartzScheduler.start(); + + quartzScheduler.setupSchedulable(new BasicSchedulableIdentity("process:test1", null), schedulableType, Collections.emptyMap(), test1.getSchedule(), false); + quartzScheduler.setupSchedulable(new BasicSchedulableIdentity("process:test2", null), schedulableType, Collections.emptyMap(), test1.getSchedule(), false); quartzScheduler.startOfSetupSchedules(); - quartzScheduler.setupProcess(test1, null, test1.getSchedule(), false); + quartzScheduler.setupSchedulable(new BasicSchedulableIdentity("process:test1", null), schedulableType, Collections.emptyMap(), test1.getSchedule(), false); quartzScheduler.endOfSetupSchedules(); List quartzJobAndTriggerWrappers = quartzScheduler.queryQuartz(); assertEquals(1, quartzJobAndTriggerWrappers.size()); - assertEquals("test1", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()); + assertEquals("process:test1", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()); } - - /******************************************************************************* - ** - *******************************************************************************/ - public static class BasicStep implements BackendStep - { - static int counter = 0; - - - - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - counter++; - } - } - } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java index 3c73cbd0..7491166f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java @@ -79,6 +79,7 @@ class QuartzJobsProcessTest extends BaseTest QSession qSession = QContext.getQSession(); QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); qScheduleManager.start(); + qScheduleManager.setupAllSchedules(); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java index f0f1af03..827cc57e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleSchedulerTest.java @@ -22,27 +22,22 @@ package com.kingsrook.qqq.backend.core.scheduler.simple; -import java.util.List; import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils; +import com.kingsrook.qqq.backend.core.scheduler.SchedulerTestUtils.BasicStep; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -75,12 +70,10 @@ class SimpleSchedulerTest extends BaseTest qScheduleManager.start(); SimpleScheduler simpleScheduler = SimpleScheduler.getInstance(qInstance); - simpleScheduler.setSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME); - simpleScheduler.start(); - assertThat(simpleScheduler.getExecutors()).isNotEmpty(); qScheduleManager.stop(); + simpleScheduler.getExecutors().forEach(e -> assertEquals(StandardScheduledExecutor.RunningState.STOPPED, e.getRunningState())); } @@ -96,16 +89,7 @@ class SimpleSchedulerTest extends BaseTest qInstance.getAutomationProviders().clear(); qInstance.getQueueProviders().clear(); - qInstance.addProcess( - new QProcessMetaData() - .withName("testScheduledProcess") - .withSchedule(new QScheduleMetaData() - .withSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME) - .withRepeatMillis(2) - .withInitialDelaySeconds(0)) - .withStepList(List.of(new QBackendStepMetaData() - .withName("step") - .withCode(new QCodeReference(BasicStep.class))))); + qInstance.addProcess(SchedulerTestUtils.buildTestProcess("testScheduledProcess", TestUtils.SIMPLE_SCHEDULER_NAME)); BasicStep.counter = 0; @@ -124,21 +108,4 @@ class SimpleSchedulerTest extends BaseTest } - - /******************************************************************************* - ** - *******************************************************************************/ - public static class BasicStep implements BackendStep - { - static int counter = 0; - - - - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - counter++; - } - } - } \ No newline at end of file From d1e4091eb4f894e74e49f75b97356aeb7fc5b482 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Mar 2024 12:28:23 -0500 Subject: [PATCH 50/72] CE-936 - Add support for managing associations on insert/edit screens, via childRecordList widget --- .../widgets/ChildRecordListRenderer.java | 127 ++++++++++++------ .../frontend/QFrontendWidgetMetaData.java | 15 +++ .../javalin/QJavalinImplementation.java | 23 +++- 3 files changed, 120 insertions(+), 45 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java index b6f69d4c..e5b2e956 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java @@ -137,7 +137,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer *******************************************************************************/ public Builder withCanAddChildRecord(boolean b) { - widgetMetaData.withDefaultValue("canAddChildRecord", true); + widgetMetaData.withDefaultValue("canAddChildRecord", b); return (this); } @@ -151,6 +151,17 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer widgetMetaData.withDefaultValue("disabledFieldsForNewChildRecords", new HashSet<>(disabledFieldsForNewChildRecords)); return (this); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withManageAssociationName(String manageAssociationName) + { + widgetMetaData.withDefaultValue("manageAssociationName", manageAssociationName); + return (this); + } } @@ -178,52 +189,60 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows")); } - //////////////////////////////////////////////////////// - // fetch the record that we're getting children for. // - // e.g., the left-side of the join, with the input id // - //////////////////////////////////////////////////////// - GetInput getInput = new GetInput(); - getInput.setTableName(join.getLeftTable()); - getInput.setPrimaryKey(id); - GetOutput getOutput = new GetAction().execute(getInput); - QRecord record = getOutput.getRecord(); - - if(record == null) + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // fetch the record that we're getting children for. e.g., the left-side of the join, with the input id // + // but - only try this if we were given an id. note, this widget could be called for on an INSERT screen, where we don't have a record yet // + // but we still want to be able to return all the other data in here that otherwise comes from the widget meta data, join, etc. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + int totalRows = 0; + QRecord primaryRecord = null; + QQueryFilter filter = null; + QueryOutput queryOutput = new QueryOutput(new QueryInput()); + if(StringUtils.hasContent(id)) { - throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id)); - } + GetInput getInput = new GetInput(); + getInput.setTableName(join.getLeftTable()); + getInput.setPrimaryKey(id); + GetOutput getOutput = new GetAction().execute(getInput); + primaryRecord = getOutput.getRecord(); - //////////////////////////////////////////////////////////////////// - // set up the query - for the table on the right side of the join // - //////////////////////////////////////////////////////////////////// - QQueryFilter filter = new QQueryFilter(); - for(JoinOn joinOn : join.getJoinOns()) - { - filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(record.getValue(joinOn.getLeftField())))); - } - filter.setOrderBys(join.getOrderBys()); - filter.setLimit(maxRows); + if(primaryRecord == null) + { + throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id)); + } - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(join.getRightTable()); - queryInput.setShouldTranslatePossibleValues(true); - queryInput.setShouldGenerateDisplayValues(true); - queryInput.setFilter(filter); - QueryOutput queryOutput = new QueryAction().execute(queryInput); + //////////////////////////////////////////////////////////////////// + // set up the query - for the table on the right side of the join // + //////////////////////////////////////////////////////////////////// + filter = new QQueryFilter(); + for(JoinOn joinOn : join.getJoinOns()) + { + filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(primaryRecord.getValue(joinOn.getLeftField())))); + } + filter.setOrderBys(join.getOrderBys()); + filter.setLimit(maxRows); - QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords()); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(join.getRightTable()); + queryInput.setShouldTranslatePossibleValues(true); + queryInput.setShouldGenerateDisplayValues(true); + queryInput.setFilter(filter); + queryOutput = new QueryAction().execute(queryInput); - int totalRows = queryOutput.getRecords().size(); - if(maxRows != null && (queryOutput.getRecords().size() == maxRows)) - { - ///////////////////////////////////////////////////////////////////////////////////// - // if the input said to only do some max, and the # of results we got is that max, // - // then do a count query, for displaying 1-n of // - ///////////////////////////////////////////////////////////////////////////////////// - CountInput countInput = new CountInput(); - countInput.setTableName(join.getRightTable()); - countInput.setFilter(filter); - totalRows = new CountAction().execute(countInput).getCount(); + QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords()); + + totalRows = queryOutput.getRecords().size(); + if(maxRows != null && (queryOutput.getRecords().size() == maxRows)) + { + ///////////////////////////////////////////////////////////////////////////////////// + // if the input said to only do some max, and the # of results we got is that max, // + // then do a count query, for displaying 1-n of // + ///////////////////////////////////////////////////////////////////////////////////// + CountInput countInput = new CountInput(); + countInput.setTableName(join.getRightTable()); + countInput.setFilter(filter); + totalRows = new CountAction().execute(countInput).getCount(); + } } String tablePath = input.getInstance().getTablePath(rightTable.getName()); @@ -239,10 +258,14 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer // new child records must have values from the join-ons // ////////////////////////////////////////////////////////// Map defaultValuesForNewChildRecords = new HashMap<>(); - for(JoinOn joinOn : join.getJoinOns()) + if(primaryRecord != null) { - defaultValuesForNewChildRecords.put(joinOn.getRightField(), record.getValue(joinOn.getLeftField())); + for(JoinOn joinOn : join.getJoinOns()) + { + defaultValuesForNewChildRecords.put(joinOn.getRightField(), primaryRecord.getValue(joinOn.getLeftField())); + } } + widgetData.setDefaultValuesForNewChildRecords(defaultValuesForNewChildRecords); Map widgetValues = input.getWidgetMetaData().getDefaultValues(); @@ -250,6 +273,22 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer { widgetData.setDisabledFieldsForNewChildRecords((Set) widgetValues.get("disabledFieldsForNewChildRecords")); } + else + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there are no disabled fields specified - then normally any fields w/ a default value get implicitly disabled // + // but - if we didn't look-up the primary record, then we'll want to explicit disable fields from joins // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(primaryRecord == null) + { + Set implicitlyDisabledFields = new HashSet<>(); + widgetData.setDisabledFieldsForNewChildRecords(implicitlyDisabledFields); + for(JoinOn joinOn : join.getJoinOns()) + { + implicitlyDisabledFields.add(joinOn.getRightField()); + } + } + } } return (new RenderWidgetOutput(widgetData)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java index 374650bf..4d5e5725 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.frontend; +import java.io.Serializable; import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonInclude; @@ -60,6 +61,7 @@ public class QFrontendWidgetMetaData protected Map icons; protected Map helpContent; + protected Map defaultValues; private final boolean hasPermission; @@ -95,6 +97,7 @@ public class QFrontendWidgetMetaData } this.helpContent = widgetMetaData.getHelpContent(); + this.defaultValues = widgetMetaData.getDefaultValues(); hasPermission = PermissionsHelper.hasWidgetPermission(actionInput, name); } @@ -274,4 +277,16 @@ public class QFrontendWidgetMetaData { return helpContent; } + + + + /******************************************************************************* + ** Getter for defaultValues + ** + *******************************************************************************/ + public Map getDefaultValues() + { + return defaultValues; + } + } diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 94ae0e71..b9b833b3 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -779,9 +779,30 @@ public class QJavalinImplementation { String fieldName = formParam.getKey(); List values = formParam.getValue(); + String value = values.get(0); + + if("associations".equals(fieldName) && StringUtils.hasContent(value)) + { + JSONObject associationsJSON = new JSONObject(value); + for(String key : associationsJSON.keySet()) + { + JSONArray associatedRecords = associationsJSON.getJSONArray(key); + for(int i = 0; i < associatedRecords.length(); i++) + { + QRecord associatedRecord = new QRecord(); + JSONObject recordJSON = associatedRecords.getJSONObject(i); + for(String k : recordJSON.keySet()) + { + associatedRecord.withValue(k, ValueUtils.getValueAsString(recordJSON.get(k))); + } + record.withAssociatedRecord(key, associatedRecord); + } + } + continue; + } + if(CollectionUtils.nullSafeHasContents(values)) { - String value = values.get(0); if(StringUtils.hasContent(value)) { record.setValue(fieldName, value); From 7b457b4936e7b6c5508f66e8e2de774bae587973 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Mar 2024 12:29:17 -0500 Subject: [PATCH 51/72] Initial checkin --- .../Aggregate2DTableWidgetRenderer.java | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java new file mode 100644 index 00000000..5a9f67ce --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java @@ -0,0 +1,202 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** Generic widget that does an aggregate query, and presents its results + ** as a table, using group-by values as both row & column labels. + *******************************************************************************/ +public class Aggregate2DTableWidgetRenderer extends AbstractWidgetRenderer +{ + private static final QLogger LOG = QLogger.getLogger(Aggregate2DTableWidgetRenderer.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + Map values = input.getWidgetMetaData().getDefaultValues(); + + String tableName = ValueUtils.getValueAsString(values.get("tableName")); + String valueField = ValueUtils.getValueAsString(values.get("valueField")); + String rowField = ValueUtils.getValueAsString(values.get("rowField")); + String columnField = ValueUtils.getValueAsString(values.get("columnField")); + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + AggregateInput aggregateInput = new AggregateInput(); + aggregateInput.setTableName(tableName); + + // todo - allow input of "list of columns" (e.g., in case some miss sometimes, or as a version of filter) + // todo - max rows, max cols? + + // todo - from input map + QQueryFilter filter = new QQueryFilter(); + aggregateInput.setFilter(filter); + + Aggregate aggregate = new Aggregate(valueField, AggregateOperator.COUNT); + aggregateInput.withAggregate(aggregate); + + GroupBy rowGroupBy = new GroupBy(table.getField(rowField)); + GroupBy columnGroupBy = new GroupBy(table.getField(columnField)); + aggregateInput.withGroupBy(rowGroupBy); + aggregateInput.withGroupBy(columnGroupBy); + + String orderBys = ValueUtils.getValueAsString(values.get("orderBys")); + if(StringUtils.hasContent(orderBys)) + { + for(String orderBy : orderBys.split(",")) + { + switch(orderBy) + { + case "row" -> filter.addOrderBy(new QFilterOrderByGroupBy(rowGroupBy)); + case "column" -> filter.addOrderBy(new QFilterOrderByGroupBy(columnGroupBy)); + case "value" -> filter.addOrderBy(new QFilterOrderByAggregate(aggregate)); + default -> LOG.warn("Unrecognized orderBy: " + orderBy); + } + } + } + + AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput); + + Map> data = new LinkedHashMap<>(); + Set columnsSet = new LinkedHashSet<>(); + + for(AggregateResult result : aggregateOutput.getResults()) + { + Serializable column = result.getGroupByValue(columnGroupBy); + Serializable row = result.getGroupByValue(rowGroupBy); + Serializable value = result.getAggregateValue(aggregate); + + Map rowMap = data.computeIfAbsent(row, (k) -> new LinkedHashMap<>()); + rowMap.put(column, value); + columnsSet.add(column); + } + + // todo - possible values from rows, cols + + //////////////////////////////////// + // setup datastructures for table // + //////////////////////////////////// + List> tableRows = new ArrayList<>(); + List tableColumns = new ArrayList<>(); + tableColumns.add(new TableData.Column("default", table.getField(rowField).getLabel(), "_row", "2fr", "left")); + + for(Serializable column : columnsSet) + { + tableColumns.add(new TableData.Column("default", String.valueOf(column) /* todo display value */, String.valueOf(column), "1fr", "right")); + } + + tableColumns.add(new TableData.Column("default", "Total", "_total", "1fr", "right")); + + TableData tableData = new TableData(null, tableColumns, tableRows) + .withRowsPerPage(100) + .withFixedStickyLastRow(false) + .withHidePaginationDropdown(true); + + Map columnSums = new HashMap<>(); + int grandTotal = 0; + for(Map.Entry> rowEntry : data.entrySet()) + { + Map rowMap = new HashMap<>(); + tableRows.add(rowMap); + + rowMap.put("_row", rowEntry.getKey() /* todo display value */); + int rowTotal = 0; + for(Serializable column : columnsSet) + { + Serializable value = rowEntry.getValue().get(column); + if(value == null) + { + value = 0; // todo? + } + + Integer valueAsInteger = Objects.requireNonNullElse(ValueUtils.getValueAsInteger(value), 0); + rowTotal += valueAsInteger; + columnSums.putIfAbsent(column, 0); + columnSums.put(column, columnSums.get(column) + valueAsInteger); + + rowMap.put(String.valueOf(column), value); // todo format commas? + } + + rowMap.put("_total", rowTotal); + grandTotal += rowTotal; + } + + /////////////// + // total row // + /////////////// + Map totalRowMap = new HashMap<>(); + tableRows.add(totalRowMap); + + totalRowMap.put("_row", "Total"); + int rowTotal = 0; + for(Serializable column : columnsSet) + { + Serializable value = columnSums.get(column); + if(value == null) + { + value = 0; // todo? + } + + totalRowMap.put(String.valueOf(column), value); // todo format commas? + } + + totalRowMap.put("_total", grandTotal); + + return (new RenderWidgetOutput(tableData)); + } + +} From 59a4ad7de601cdb45c1e1bec0280c078ca6b22ee Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Mar 2024 12:49:26 -0500 Subject: [PATCH 52/72] Checkstyle --- .../qqq/backend/core/actions/customizers/QCodeLoader.java | 3 ++- .../qqq/backend/core/logging/CollectedLogMessage.java | 6 +----- .../qqq/backend/core/scheduler/quartz/QuartzJobRunner.java | 2 +- .../qqq/backend/core/scheduler/simple/SimpleJobRunner.java | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java index a8adaed6..14773be9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java @@ -169,7 +169,8 @@ public class QCodeLoader try { - Optional> constructor = constructorMemoization.getResultThrowing(codeReference.getName(), (UnsafeFunction, Exception>) s -> { + Optional> constructor = constructorMemoization.getResultThrowing(codeReference.getName(), (UnsafeFunction, Exception>) s -> + { Class customizerClass = Class.forName(codeReference.getName()); return customizerClass.getConstructor(); }); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java index ff607b8e..635a7410 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java @@ -54,11 +54,7 @@ public class CollectedLogMessage @Override public String toString() { - return "CollectedLogMessage{" + - "level=" + level + - ", message='" + message + '\'' + - ", exception=" + exception + - '}'; + return "CollectedLogMessage{level=" + level + ", message='" + message + '\'' + ", exception=" + exception + '}'; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java index c76b7d60..64831ac9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java @@ -28,8 +28,8 @@ import com.kingsrook.qqq.backend.core.context.CapturedContext; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java index 29e6e976..bbdf6466 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/simple/SimpleJobRunner.java @@ -28,8 +28,8 @@ import com.kingsrook.qqq.backend.core.context.CapturedContext; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; From 2088c5dab37f7df322a430edcac731c93e8f965e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Mar 2024 15:03:42 -0500 Subject: [PATCH 53/72] Fixing tests --- .../qqq/backend/core/scheduler/QScheduleManagerTest.java | 2 ++ .../scheduler/quartz/processes/QuartzJobsProcessTest.java | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java index d5e97c60..cef9d263 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java @@ -186,11 +186,13 @@ class QScheduleManagerTest extends BaseTest .withId(2) .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + qInstance.getQueue(TestUtils.TEST_SQS_QUEUE).setSchedule(null); qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.QUEUE_PROCESSOR, Map.of("queueName", TestUtils.TEST_SQS_QUEUE)) .withId(3) .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME)); + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().setSchedule(null); qScheduleManager.setupScheduledJob(newScheduledJob(ScheduledJobType.TABLE_AUTOMATIONS, Map.of("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY, "automationStatus", AutomationStatus.PENDING_UPDATE_AUTOMATIONS.name())) .withId(4) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java index 7491166f..5ce1c175 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.scheduler.quartz.processes; import java.util.List; +import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; @@ -43,6 +44,7 @@ import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzScheduler; import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -132,6 +134,7 @@ class QuartzJobsProcessTest extends BaseTest /////////////////////////////// RunProcessInput input = new RunProcessInput(); input.setProcessName(PauseAllQuartzJobsProcess.class.getSimpleName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); new RunProcessAction().execute(input); ////////////////////////////////////// @@ -153,8 +156,11 @@ class QuartzJobsProcessTest extends BaseTest /////////////////////////////// RunProcessInput input = new RunProcessInput(); input.setProcessName(PauseAllQuartzJobsProcess.class.getSimpleName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); new RunProcessAction().execute(input); + SleepUtils.sleep(3, TimeUnit.SECONDS); + ////////////////////////////////////// // assert everything becomes paused // ////////////////////////////////////// @@ -165,6 +171,7 @@ class QuartzJobsProcessTest extends BaseTest //////////////////// input = new RunProcessInput(); input.setProcessName(ResumeAllQuartzJobsProcess.class.getSimpleName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); new RunProcessAction().execute(input); //////////////////////////////////////// @@ -198,6 +205,7 @@ class QuartzJobsProcessTest extends BaseTest ////////////////////////// input = new RunProcessInput(); input.setProcessName(ResumeAllQuartzJobsProcess.class.getSimpleName()); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); new RunProcessAction().execute(input); //////////////////////////////////////// From 04103281afe343c9286f3dd18b6090c13c731e03 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Mar 2024 15:11:46 -0500 Subject: [PATCH 54/72] Fix iteration over form params (changed w/ addition of associations) - to handle empty values list --- .../javalin/QJavalinImplementation.java | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index b9b833b3..62332661 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -779,30 +779,31 @@ public class QJavalinImplementation { String fieldName = formParam.getKey(); List values = formParam.getValue(); - String value = values.get(0); - - if("associations".equals(fieldName) && StringUtils.hasContent(value)) - { - JSONObject associationsJSON = new JSONObject(value); - for(String key : associationsJSON.keySet()) - { - JSONArray associatedRecords = associationsJSON.getJSONArray(key); - for(int i = 0; i < associatedRecords.length(); i++) - { - QRecord associatedRecord = new QRecord(); - JSONObject recordJSON = associatedRecords.getJSONObject(i); - for(String k : recordJSON.keySet()) - { - associatedRecord.withValue(k, ValueUtils.getValueAsString(recordJSON.get(k))); - } - record.withAssociatedRecord(key, associatedRecord); - } - } - continue; - } if(CollectionUtils.nullSafeHasContents(values)) { + String value = values.get(0); + + if("associations".equals(fieldName) && StringUtils.hasContent(value)) + { + JSONObject associationsJSON = new JSONObject(value); + for(String key : associationsJSON.keySet()) + { + JSONArray associatedRecords = associationsJSON.getJSONArray(key); + for(int i = 0; i < associatedRecords.length(); i++) + { + QRecord associatedRecord = new QRecord(); + JSONObject recordJSON = associatedRecords.getJSONObject(i); + for(String k : recordJSON.keySet()) + { + associatedRecord.withValue(k, ValueUtils.getValueAsString(recordJSON.get(k))); + } + record.withAssociatedRecord(key, associatedRecord); + } + } + continue; + } + if(StringUtils.hasContent(value)) { record.setValue(fieldName, value); @@ -814,7 +815,6 @@ public class QJavalinImplementation } else { - // is this ever hit? record.setValue(fieldName, null); } } From a10992226a111f66aea6eb9189d24fdc8f3ec979 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 10:08:11 -0500 Subject: [PATCH 55/72] CE-936 - Cleanup from code review --- .../core/actions/async/AsyncJobManager.java | 1 - .../SchedulableTableAutomationsRunner.java | 4 -- .../src/main/resources/quartz.properties | 64 ------------------- 3 files changed, 69 deletions(-) delete mode 100644 qqq-backend-core/src/main/resources/quartz.properties diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java index ae980ea1..6cc317d5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java @@ -159,7 +159,6 @@ public class AsyncJobManager private T runAsyncJob(String jobName, AsyncJob asyncJob, UUIDAndTypeStateKey uuidAndTypeStateKey, AsyncJobStatus asyncJobStatus) { String originalThreadName = Thread.currentThread().getName(); - // Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8)); Thread.currentThread().setName("Job:" + jobName); try { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java index 5a6e0e53..44bd2f20 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/runner/SchedulableTableAutomationsRunner.java @@ -75,10 +75,6 @@ public class SchedulableTableAutomationsRunner implements SchedulableRunner AutomationStatus automationStatus = AutomationStatus.valueOf(ValueUtils.getValueAsString(params.get("automationStatus"))); - ///////////// - // run it. // - ///////////// - LOG.debug("Running table automations", logPair("tableName", tableName), logPair("")); QTableAutomationDetails automationDetails = table.getAutomationDetails(); if(automationDetails == null) { diff --git a/qqq-backend-core/src/main/resources/quartz.properties b/qqq-backend-core/src/main/resources/quartz.properties deleted file mode 100644 index 3dd81314..00000000 --- a/qqq-backend-core/src/main/resources/quartz.properties +++ /dev/null @@ -1,64 +0,0 @@ -# -# QQQ - Low-code Application Framework for Engineers. -# Copyright (C) 2021-2023. Kingsrook, LLC -# 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States -# contact@kingsrook.com -# https://github.com/Kingsrook/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -# -#org.quartz.scheduler.instanceName = MyScheduler -#org.quartz.threadPool.threadCount = 3 -#org.quartz.jobStore.class = org.quartz.simpl.RAMJobStore -# - - - -#============================================================================ -# Configure Main Scheduler Properties -#============================================================================ -org.quartz.scheduler.instanceName = MyClusteredScheduler -org.quartz.scheduler.instanceId = AUTO - -#============================================================================ -# Configure ThreadPool -#============================================================================ -org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool -org.quartz.threadPool.threadCount = 5 -org.quartz.threadPool.threadPriority = 5 - -#============================================================================ -# Configure JobStore -#============================================================================ -org.quartz.jobStore.misfireThreshold = 60000 - -org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX -org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate -org.quartz.jobStore.useProperties = false -org.quartz.jobStore.dataSource = myDS -org.quartz.jobStore.tablePrefix = QUARTZ_ - -org.quartz.jobStore.isClustered = true -org.quartz.jobStore.clusterCheckinInterval = 20000 - -#============================================================================ -# Configure Datasources -#============================================================================ -org.quartz.dataSource.myDS.driver = com.mysql.cj.jdbc.Driver -org.quartz.dataSource.myDS.URL = jdbc:mysql://localhost:3306/nutrifresh_one -org.quartz.dataSource.myDS.user = root -org.quartz.dataSource.myDS.password = BXca6Bubxf!ECt7sua6L -org.quartz.dataSource.myDS.maxConnections = 5 -org.quartz.dataSource.myDS.validationQuery=select 1 \ No newline at end of file From d6edbfa06beeb51ba15b574b38a267c7d5480e30 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 10:08:25 -0500 Subject: [PATCH 56/72] CE-936 - Add exposed joins between these tables --- .../scheduledjobs/ScheduledJobsMetaDataProvider.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java index 352b5de1..9b10da45 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRule import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -183,6 +184,11 @@ public class ScheduledJobsMetaDataProvider .withAssociatedTableName(ScheduledJobParameter.TABLE_NAME) .withJoinName(JOB_PARAMETER_JOIN_NAME)); + tableMetaData.withExposedJoin(new ExposedJoin() + .withJoinTable(ScheduledJobParameter.TABLE_NAME) + .withJoinPath(List.of(JOB_PARAMETER_JOIN_NAME)) + .withLabel("Parameters")); + return (tableMetaData); } @@ -199,6 +205,11 @@ public class ScheduledJobsMetaDataProvider .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "scheduledJobId", "key", "value"))) .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + tableMetaData.withExposedJoin(new ExposedJoin() + .withJoinTable(ScheduledJob.TABLE_NAME) + .withJoinPath(List.of(JOB_PARAMETER_JOIN_NAME)) + .withLabel("Scheduled Job")); + return (tableMetaData); } From 7015322bf37a8a08c1dd6769e3eded8c7ec15d93 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 10:09:28 -0500 Subject: [PATCH 57/72] CE-936 - format epoc time values w/ commas and an instant in system time zone --- .../QuartzJobDataPostQueryCustomizer.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDataPostQueryCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDataPostQueryCustomizer.java index 372a51e3..f16cbb83 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDataPostQueryCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/tables/QuartzJobDataPostQueryCustomizer.java @@ -22,12 +22,17 @@ package com.kingsrook.qqq.backend.core.scheduler.quartz.tables; +import java.time.Instant; +import java.time.ZoneId; import java.util.List; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import org.apache.commons.lang3.SerializationUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -67,8 +72,35 @@ public class QuartzJobDataPostQueryCustomizer extends AbstractPostQueryCustomize LOG.info("Error deserializing quartz job data", e); } } + + formatEpochTime(record, "nextFireTime"); + formatEpochTime(record, "prevFireTime"); + formatEpochTime(record, "startTime"); } return (records); } + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void formatEpochTime(QRecord record, String fieldName) + { + Long value = record.getValueLong(fieldName); + + try + { + if(value != null && value > 0) + { + Instant instant = Instant.ofEpochMilli(value); + record.setDisplayValue(fieldName, String.format("%,d", value) + " (" + QValueFormatter.formatDateTimeWithZone(instant.atZone(ZoneId.of(QContext.getQInstance().getDefaultTimeZoneId()))) + ")"); + } + } + catch(Exception e) + { + LOG.info("Error formatting an epoc time value", e, logPair("fieldName", fieldName), logPair("value", value)); + } + } + } From c8c70516281e3a62bad0b48baca4af24c5435f9e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 11:32:41 -0500 Subject: [PATCH 58/72] CE-936 - Update to send warnings from insert & update back not as an exception, but as a success, with warnings in the record. --- .../javalin/QJavalinImplementation.java | 27 ++++-- .../javalin/QJavalinImplementationTest.java | 82 +++++++++++++++++++ .../qqq/backend/javalin/TestUtils.java | 54 ++++++++++++ 3 files changed, 155 insertions(+), 8 deletions(-) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 62332661..6b674b09 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -710,10 +710,15 @@ public class QJavalinImplementation { throw (new QUserFacingException("Error updating " + tableMetaData.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getErrors()))); } - if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings())) - { - throw (new QUserFacingException("Warning updating " + tableMetaData.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getWarnings()))); - } + + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // at one time, we threw upon warning - but // + // on insert we need to return the record (e.g., to get a generated id), so, make update do the same. // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings())) + // { + // throw (new QUserFacingException("Warning updating " + tableMetaData.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getWarnings()))); + // } QJavalinAccessLogger.logEndSuccess(); context.result(JsonUtils.toJson(updateOutput)); @@ -902,10 +907,16 @@ public class QJavalinImplementation { throw (new QUserFacingException("Error inserting " + table.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getErrors()))); } - if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings())) - { - throw (new QUserFacingException("Warning inserting " + table.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getWarnings()))); - } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // at one time, we threw upon warning - but // + // our use-case is, the frontend, it wants to get the record, and show a success (with the generated id) // + // and then to also show a warning message - so - let it all be returned and handled on the frontend. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings())) + // { + // throw (new QUserFacingException("Warning inserting " + table.getLabel() + ": " + joinErrorsWithCommasAndAnd(outputRecord.getWarnings()))); + // } QJavalinAccessLogger.logEndSuccess(logPair("primaryKey", () -> (outputRecord.getValue(table.getPrimaryKeyField())))); context.result(JsonUtils.toJson(insertOutput)); diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java index a352e558..76a879b0 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java @@ -514,6 +514,42 @@ class QJavalinImplementationTest extends QJavalinTestBase + /******************************************************************************* + ** test an insert that returns a warning + ** + *******************************************************************************/ + @Test + public void test_dataInsertWithWarning() + { + Map body = new HashMap<>(); + body.put("firstName", "Warning"); + body.put("lastName", "Kelkhoff"); + body.put("email", "warning@kelkhoff.com"); + + HttpResponse response = Unirest.post(BASE_URL + "/data/person") + .header("Content-Type", "application/json") + .body(body) + .asString(); + + assertEquals(200, response.getStatus()); + JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody()); + assertTrue(jsonObject.has("records")); + JSONArray records = jsonObject.getJSONArray("records"); + assertEquals(1, records.length()); + JSONObject record0 = records.getJSONObject(0); + assertTrue(record0.has("values")); + JSONObject values0 = record0.getJSONObject("values"); + assertTrue(values0.has("id")); + assertEquals(7, values0.getInt("id")); + + assertTrue(record0.has("warnings")); + JSONArray warnings = record0.getJSONArray("warnings"); + assertEquals(1, warnings.length()); + assertTrue(warnings.getJSONObject(0).has("message")); + } + + + /******************************************************************************* ** test an insert - posting a multipart form. ** @@ -594,6 +630,52 @@ class QJavalinImplementationTest extends QJavalinTestBase + /******************************************************************************* + ** test an update - with a warning returned + ** + *******************************************************************************/ + @Test + public void test_dataUpdateWithWarning() + { + Map body = new HashMap<>(); + body.put("firstName", "Warning"); + body.put("birthDate", ""); + + HttpResponse response = Unirest.patch(BASE_URL + "/data/person/4") + .header("Content-Type", "application/json") + .body(body) + .asString(); + + assertEquals(200, response.getStatus()); + JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody()); + assertTrue(jsonObject.has("records")); + JSONArray records = jsonObject.getJSONArray("records"); + assertEquals(1, records.length()); + JSONObject record0 = records.getJSONObject(0); + assertTrue(record0.has("values")); + assertEquals("person", record0.getString("tableName")); + JSONObject values0 = record0.getJSONObject("values"); + assertEquals(4, values0.getInt("id")); + assertEquals("Warning", values0.getString("firstName")); + + assertTrue(record0.has("warnings")); + JSONArray warnings = record0.getJSONArray("warnings"); + assertEquals(1, warnings.length()); + assertTrue(warnings.getJSONObject(0).has("message")); + + /////////////////////////////////////////////////////////////////// + // re-GET the record, and validate that birthDate was nulled out // + /////////////////////////////////////////////////////////////////// + response = Unirest.get(BASE_URL + "/data/person/4").asString(); + assertEquals(200, response.getStatus()); + jsonObject = JsonUtils.toJSONObject(response.getBody()); + assertTrue(jsonObject.has("values")); + JSONObject values = jsonObject.getJSONObject("values"); + assertFalse(values.has("birthDate")); + } + + + /******************************************************************************* ** test an update - posting the data as a multipart form ** diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java index 430f72ae..ae5d2694 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java @@ -25,6 +25,10 @@ package com.kingsrook.qqq.backend.javalin; import java.io.InputStream; import java.sql.Connection; import java.util.List; +import java.util.Objects; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -35,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -68,6 +73,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.savedviews.SavedViewsMetaDataProvider; import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; @@ -255,6 +261,9 @@ public class TestUtils .withScriptTypeId(1) .withScriptTester(new QCodeReference(TestScriptAction.class))); + qTableMetaData.withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(PersonTableCustomizer.class)); + qTableMetaData.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(PersonTableCustomizer.class)); + qTableMetaData.getField("photo") .withIsHeavy(true) .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD) @@ -265,6 +274,51 @@ public class TestUtils } + /******************************************************************************* + ** + *******************************************************************************/ + public static class PersonTableCustomizer implements TableCustomizerInterface + { + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + return warnPostInsertOrUpdate(records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + return warnPostInsertOrUpdate(records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List warnPostInsertOrUpdate(List records) + { + for(QRecord record : records) + { + if(Objects.requireNonNullElse(record.getValueString("firstName"), "").toLowerCase().contains("warn")) + { + record.addWarning(new QWarningMessage("Warning in firstName.")); + } + } + + return records; + } + } + + /******************************************************************************* ** From 5963a706b0a39128391313e864c1c389db69aa8f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 11:33:12 -0500 Subject: [PATCH 59/72] CE-936 - Add error if cron or repeat-seconds isn't given; add warnings from scheduling. --- .../ScheduledJobTableCustomizer.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java index 0f4a4558..f3cc54cf 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java @@ -43,6 +43,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -126,6 +127,13 @@ public class ScheduledJobTableCustomizer implements TableCustomizerInterface record.addError(new BadInputStatusMessage("If a Cron Expression is given, then a Cron Time Zone Id is required.")); } } + else + { + if(!StringUtils.hasContent(record.getValueString("repeatSeconds"))) + { + record.addError(new BadInputStatusMessage("Either Cron Expression or Repeat Seconds must be given.")); + } + } } } @@ -189,7 +197,8 @@ public class ScheduledJobTableCustomizer implements TableCustomizerInterface try { - List freshRecordListWithAssociations = freshlyQueryForRecordsWithAssociations(recordsWithoutErrors); + Map originalRecordMap = recordsWithoutErrors.stream().collect(Collectors.toMap(r -> r.getValueInteger("id"), r -> r)); + List freshRecordListWithAssociations = freshlyQueryForRecordsWithAssociations(recordsWithoutErrors); QScheduleManager scheduleManager = QScheduleManager.getInstance(); for(QRecord record : freshRecordListWithAssociations) @@ -200,7 +209,11 @@ public class ScheduledJobTableCustomizer implements TableCustomizerInterface } catch(Exception e) { - LOG.info("Caught exception while scheduling a job in post-action", e, logPair("id", record.getValue("id"))); + LOG.warn("Caught exception while scheduling a job in post-action", e, logPair("id", record.getValue("id"))); + if(originalRecordMap.containsKey(record.getValueInteger("id"))) + { + originalRecordMap.get(record.getValueInteger("id")).addWarning(new QWarningMessage("Error scheduling job: " + e.getMessage())); + } } } } @@ -265,7 +278,7 @@ public class ScheduledJobTableCustomizer implements TableCustomizerInterface } catch(Exception e) { - LOG.info("Caught exception while scheduling a job in post-action", e, logPair("id", record.getValue("id"))); + LOG.warn("Caught exception while un-scheduling a job in post-action", e, logPair("id", record.getValue("id"))); } } } From 605578d661bbd3d7737bebffe1d5bb7e4014967b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 19:59:34 -0500 Subject: [PATCH 60/72] add overload of recordsToMap that takes key type --- .../backend/core/utils/CollectionUtils.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java index 95b8d091..1e53c346 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java @@ -518,6 +518,24 @@ public class CollectionUtils + /******************************************************************************* + ** Convert a collection of QRecords to a map, from one field's values out of + ** those records, to the records themselves. + *******************************************************************************/ + public static Map recordsToMap(Collection records, String keyFieldName, Class type) + { + Map rs = new HashMap<>(); + + for(QRecord record : nonNullCollection(records)) + { + rs.put(ValueUtils.getValueAsType(type, record.getValue(keyFieldName)), record); + } + + return (rs); + } + + + /******************************************************************************* ** *******************************************************************************/ From 17899c3fdcf866e6591949d2411bfd24cd28aa7b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 20:00:24 -0500 Subject: [PATCH 61/72] CE-936 - Move per-record pause & resume to run on quartzJob tables, not trigger; update previewMessage --- .../scheduler/quartz/processes/PauseQuartzJobsProcess.java | 3 ++- .../scheduler/quartz/processes/ResumeQuartzJobsProcess.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java index f74791cc..c5d3d38b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/PauseQuartzJobsProcess.java @@ -52,11 +52,12 @@ public class PauseQuartzJobsProcess extends AbstractLoadStep implements MetaData @Override public QProcessMetaData produce(QInstance qInstance) throws QException { - String tableName = "quartzTriggers"; + String tableName = "quartzJobDetails"; return StreamedETLWithFrontendProcess.processMetaDataBuilder() .withName(getClass().getSimpleName()) .withLabel("Pause Quartz Jobs") + .withPreviewMessage("This is a preview of the jobs that will be paused.") .withTableName(tableName) .withSourceTable(tableName) .withDestinationTable(tableName) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java index 687f29b3..c4a8f3b8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/ResumeQuartzJobsProcess.java @@ -52,11 +52,12 @@ public class ResumeQuartzJobsProcess extends AbstractLoadStep implements MetaDat @Override public QProcessMetaData produce(QInstance qInstance) throws QException { - String tableName = "quartzTriggers"; + String tableName = "quartzJobDetails"; return StreamedETLWithFrontendProcess.processMetaDataBuilder() .withName(getClass().getSimpleName()) .withLabel("Resume Quartz Jobs") + .withPreviewMessage("This is a preview of the jobs that will be resumed.") .withTableName(tableName) .withSourceTable(tableName) .withDestinationTable(tableName) From 24b1daa110f08f83ad7cf0f5edba0be0760e96c2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 20:00:49 -0500 Subject: [PATCH 62/72] CE-936 - Move adding default schedulableTypes in instance to Enricher --- .../backend/core/instances/QInstanceEnricher.java | 13 +++++++++++++ .../backend/core/scheduler/QScheduleManager.java | 9 --------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 45d249ed..4bcac23f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -77,6 +77,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.Bulk import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -159,6 +160,18 @@ public class QInstanceEnricher } enrichJoins(); + + ////////////////////////////////////////////////////////////////////////////// + // if the instance DOES have 1 or more scheduler, but no schedulable types, // + // then go ahead and add the default set that qqq knows about // + ////////////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(qInstance.getSchedulers())) + { + if(CollectionUtils.nullSafeIsEmpty(qInstance.getSchedulableTypes())) + { + QScheduleManager.defineDefaultSchedulableTypesInInstance(qInstance); + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index 747c366b..1d8991b1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -102,15 +102,6 @@ public class QScheduleManager { qScheduleManager = new QScheduleManager(qInstance, systemUserSessionSupplier); - ///////////////////////////////////////////////////////////////// - // if the instance doesn't have any schedulable types defined, // - // then go ahead and add the default set that qqq knows about // - ///////////////////////////////////////////////////////////////// - if(CollectionUtils.nullSafeIsEmpty(qInstance.getSchedulableTypes())) - { - defineDefaultSchedulableTypesInInstance(qInstance); - } - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // initialize the scheduler(s) we're configured to use // // do this, even if we won't start them - so, for example, a web server can still be aware of schedules in the application // From 8afbbfb4daf4f9bd90c55a89bd4e28ef647a3c3f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 20:01:08 -0500 Subject: [PATCH 63/72] Avoid NPE on null value for custom-type PVS --- .../core/actions/values/QPossibleValueTranslator.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java index 4626ac8d..a6a23d86 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java @@ -370,6 +370,14 @@ public class QPossibleValueTranslator *******************************************************************************/ private String translatePossibleValueCustom(Serializable value, QPossibleValueSource possibleValueSource) { + ///////////////////////////////// + // null input gets null output // + ///////////////////////////////// + if(value == null) + { + return (null); + } + try { QCustomPossibleValueProvider customPossibleValueProvider = QCodeLoader.getCustomPossibleValueProvider(possibleValueSource); From 92b8211f20bcbdd716e41384bfb84533416099fa Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 20:02:13 -0500 Subject: [PATCH 64/72] CE-936 Add mayUseInScheduledJobsTable to scheduler meta data; check that before including in Schedulers PVS --- .../model/metadata/scheduleing/QSchedulerMetaData.java | 10 ++++++++++ .../scheduleing/quartz/QuartzSchedulerMetaData.java | 10 ++++++++++ .../scheduleing/simple/SimpleSchedulerMetaData.java | 10 ++++++++++ .../scheduledjobs/SchedulersPossibleValueSource.java | 5 ++++- 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java index 90fb8f92..9eda36b2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QSchedulerMetaData.java @@ -50,6 +50,16 @@ public abstract class QSchedulerMetaData implements TopLevelMetaDataInterface + /******************************************************************************* + ** + *******************************************************************************/ + public boolean mayUseInScheduledJobsTable() + { + return (true); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java index 12ef0361..3367cd25 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/quartz/QuartzSchedulerMetaData.java @@ -68,6 +68,16 @@ public class QuartzSchedulerMetaData extends QSchedulerMetaData + /******************************************************************************* + ** + *******************************************************************************/ + public boolean mayUseInScheduledJobsTable() + { + return (true); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java index 92ef2c99..69882b41 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/simple/SimpleSchedulerMetaData.java @@ -61,6 +61,16 @@ public class SimpleSchedulerMetaData extends QSchedulerMetaData + /******************************************************************************* + ** + *******************************************************************************/ + public boolean mayUseInScheduledJobsTable() + { + return (false); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java index a2363267..e46c81c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/SchedulersPossibleValueSource.java @@ -69,7 +69,10 @@ public class SchedulersPossibleValueSource implements QCustomPossibleValueProvid List> rs = new ArrayList<>(); for(QSchedulerMetaData scheduler : CollectionUtils.nonNullMap(QContext.getQInstance().getSchedulers()).values()) { - rs.add(schedulerToPossibleValue(scheduler)); + if(scheduler.mayUseInScheduledJobsTable()) + { + rs.add(schedulerToPossibleValue(scheduler)); + } } return rs; } From 2545d03f20294c356c6c99800d683fda5c4c8d27 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 20:04:05 -0500 Subject: [PATCH 65/72] CE-936 Switch scheduledJobTypes PVS to be based on schedulable types in the instance, not just the enum (e.g., in support of app-defined types) --- .../model/scheduledjobs/ScheduledJob.java | 2 +- .../model/scheduledjobs/ScheduledJobType.java | 98 ++----------------- .../ScheduledJobTypePossibleValueSource.java | 87 ++++++++++++++++ .../ScheduledJobsMetaDataProvider.java | 16 ++- .../core/scheduler/QScheduleManager.java | 18 ++-- .../identity/SchedulableIdentityFactory.java | 12 +-- .../core/scheduler/QScheduleManagerTest.java | 2 +- .../scheduler/quartz/QuartzSchedulerTest.java | 3 +- 8 files changed, 123 insertions(+), 115 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobTypePossibleValueSource.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java index 54bfc182..e923d549 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJob.java @@ -73,7 +73,7 @@ public class ScheduledJob extends QRecordEntity @QField(displayFormat = DisplayFormat.COMMAS) private Integer repeatSeconds; - @QField(isRequired = true, maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = ScheduledJobType.NAME) + @QField(isRequired = true, maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.ERROR, possibleValueSourceName = ScheduledJobTypePossibleValueSource.NAME) private String type; @QField(isRequired = true) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java index c8296e40..56114667 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobType.java @@ -22,102 +22,16 @@ package com.kingsrook.qqq.backend.core.model.scheduledjobs; -import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; -import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; - - /******************************************************************************* + ** enum of core schedulable types that QQQ schedule manager directly knows about. ** + ** note though, that applications can define their own schedulable types, + ** by adding SchedulableType meta-data to the QInstance, and providing classes + ** that implement SchedulableRunner. *******************************************************************************/ -public enum ScheduledJobType implements PossibleValueEnum +public enum ScheduledJobType { PROCESS, QUEUE_PROCESSOR, - TABLE_AUTOMATIONS, - // todo - future - USER_REPORT - ; - - public static final String NAME = "scheduledJobType"; - - private final String label; - - - - /******************************************************************************* - ** Constructor - ** - *******************************************************************************/ - ScheduledJobType() - { - this.label = QInstanceEnricher.nameToLabel(QInstanceEnricher.inferNameFromBackendName(name())); - } - - - - /******************************************************************************* - ** Get instance by id - ** - *******************************************************************************/ - public static ScheduledJobType getById(String id) - { - if(id == null) - { - return (null); - } - - for(ScheduledJobType value : ScheduledJobType.values()) - { - if(value.name().equals(id)) - { - return (value); - } - } - - return (null); - } - - - - /******************************************************************************* - ** Getter for id - ** - *******************************************************************************/ - public String getId() - { - return name(); - } - - - - /******************************************************************************* - ** Getter for label - ** - *******************************************************************************/ - public String getLabel() - { - return label; - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public String getPossibleValueId() - { - return name(); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public String getPossibleValueLabel() - { - return (label); - } - + TABLE_AUTOMATIONS } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobTypePossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobTypePossibleValueSource.java new file mode 100644 index 00000000..43d26dd5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobTypePossibleValueSource.java @@ -0,0 +1,87 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobTypePossibleValueSource implements QCustomPossibleValueProvider +{ + public static final String NAME = "scheduledJobType"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QPossibleValue getPossibleValue(Serializable idValue) + { + SchedulableType schedulableType = QContext.getQInstance().getSchedulableType(String.valueOf(idValue)); + if(schedulableType != null) + { + return schedulableTypeToPossibleValue(schedulableType); + } + + return null; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List> search(SearchPossibleValueSourceInput input) throws QException + { + List> rs = new ArrayList<>(); + for(SchedulableType schedulableType : CollectionUtils.nonNullMap(QContext.getQInstance().getSchedulableTypes()).values()) + { + rs.add(schedulableTypeToPossibleValue(schedulableType)); + } + return rs; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QPossibleValue schedulableTypeToPossibleValue(SchedulableType schedulableType) + { + return new QPossibleValue<>(schedulableType.getName(), schedulableType.getName()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java index 9b10da45..2e0036ae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java @@ -64,7 +64,7 @@ public class ScheduledJobsMetaDataProvider { defineStandardTables(instance, backendName, backendDetailEnricher); instance.addPossibleValueSource(QPossibleValueSource.newForTable(ScheduledJob.TABLE_NAME)); - instance.addPossibleValueSource(QPossibleValueSource.newForEnum(ScheduledJobType.NAME, ScheduledJobType.values())); + instance.addPossibleValueSource(defineScheduledJobTypePossibleValueSource()); instance.addPossibleValueSource(defineSchedulersPossibleValueSource()); defineStandardJoins(instance); defineStandardWidgets(instance); @@ -215,6 +215,19 @@ public class ScheduledJobsMetaDataProvider + + /******************************************************************************* + ** + *******************************************************************************/ + private QPossibleValueSource defineScheduledJobTypePossibleValueSource() + { + return (new QPossibleValueSource() + .withName(ScheduledJobTypePossibleValueSource.NAME) + .withType(QPossibleValueSourceType.CUSTOM) + .withCustomCodeReference(new QCodeReference(ScheduledJobTypePossibleValueSource.class))); + } + + /******************************************************************************* ** *******************************************************************************/ @@ -224,7 +237,6 @@ public class ScheduledJobsMetaDataProvider .withName(SchedulersPossibleValueSource.NAME) .withType(QPossibleValueSourceType.CUSTOM) .withCustomCodeReference(new QCodeReference(SchedulersPossibleValueSource.class))); - } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java index 1d8991b1..a51999ab 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java @@ -122,9 +122,9 @@ public class QScheduleManager *******************************************************************************/ public static void defineDefaultSchedulableTypesInInstance(QInstance qInstance) { - qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.PROCESS.getId()).withRunner(new QCodeReference(SchedulableProcessRunner.class))); - qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.QUEUE_PROCESSOR.getId()).withRunner(new QCodeReference(SchedulableSQSQueueRunner.class))); - qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.TABLE_AUTOMATIONS.getId()).withRunner(new QCodeReference(SchedulableTableAutomationsRunner.class))); + qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.PROCESS.name()).withRunner(new QCodeReference(SchedulableProcessRunner.class))); + qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.QUEUE_PROCESSOR.name()).withRunner(new QCodeReference(SchedulableSQSQueueRunner.class))); + qInstance.addSchedulableType(new SchedulableType().withName(ScheduledJobType.TABLE_AUTOMATIONS.name()).withRunner(new QCodeReference(SchedulableTableAutomationsRunner.class))); } @@ -321,8 +321,8 @@ public class QScheduleManager throw (new QException("Missing a type " + exceptionSuffix)); } - ScheduledJobType scheduledJobType = ScheduledJobType.getById(scheduledJob.getType()); - if(scheduledJobType == null) + SchedulableType schedulableType = qInstance.getSchedulableType(scheduledJob.getType()); + if(schedulableType == null) { throw (new QException("Unrecognized type [" + scheduledJob.getType() + "] " + exceptionSuffix)); } @@ -330,8 +330,6 @@ public class QScheduleManager QSchedulerInterface scheduler = getScheduler(scheduledJob.getSchedulerName()); Map paramMap = new HashMap<>(scheduledJob.getJobParametersMap()); - SchedulableType schedulableType = qInstance.getSchedulableType(scheduledJob.getType()); - SchedulableRunner runner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); runner.validateParams(schedulableIdentity, new HashMap<>(paramMap)); @@ -387,7 +385,7 @@ public class QScheduleManager Map paramMap = new HashMap<>(); paramMap.put("processName", process.getName()); - SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.PROCESS.getId()); + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.PROCESS.name()); if(process.getVariantBackend() == null || VariantRunStrategy.SERIAL.equals(process.getVariantRunStrategy())) { @@ -437,7 +435,7 @@ public class QScheduleManager *******************************************************************************/ private void setupTableAutomations(QTableMetaData table) throws QException { - SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.TABLE_AUTOMATIONS.getId()); + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.TABLE_AUTOMATIONS.name()); QTableAutomationDetails automationDetails = table.getAutomationDetails(); QSchedulerInterface scheduler = getScheduler(automationDetails.getSchedule().getSchedulerName()); @@ -466,7 +464,7 @@ public class QScheduleManager { SchedulableIdentity schedulableIdentity = SchedulableIdentityFactory.of(queue); QSchedulerInterface scheduler = getScheduler(queue.getSchedule().getSchedulerName()); - SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.QUEUE_PROCESSOR.getId()); + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.QUEUE_PROCESSOR.name()); boolean allowedToStart = SchedulerUtils.allowedToStart(queue.getName()); Map paramMap = new HashMap<>(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java index 01b1d17b..0ad89408 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/schedulable/identity/SchedulableIdentityFactory.java @@ -29,7 +29,6 @@ import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; -import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; @@ -45,19 +44,18 @@ public class SchedulableIdentityFactory *******************************************************************************/ public static BasicSchedulableIdentity of(ScheduledJob scheduledJob) { - String description = ""; - ScheduledJobType scheduledJobType = ScheduledJobType.getById(scheduledJob.getType()); - if(scheduledJobType != null) + String description = ""; + SchedulableType schedulableType = QContext.getQInstance().getSchedulableType(scheduledJob.getType()); + if(schedulableType != null) { try { - SchedulableType schedulableType = QContext.getQInstance().getSchedulableType(scheduledJob.getType()); - SchedulableRunner runner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + SchedulableRunner runner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); description = runner.getDescription(new HashMap<>(scheduledJob.getJobParametersMap())); } catch(Exception e) { - description = "type: " + scheduledJobType; + description = "type: " + schedulableType.getName(); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java index cef9d263..68a7c37b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManagerTest.java @@ -92,7 +92,7 @@ class QScheduleManagerTest extends BaseTest .withId(1) .withIsActive(true) .withSchedulerName(TestUtils.SIMPLE_SCHEDULER_NAME) - .withType(type.getId()) + .withType(type.name()) .withRepeatSeconds(1) .withJobParameters(new ArrayList<>()); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java index 43fe0b1d..f83972c2 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java @@ -156,7 +156,6 @@ class QuartzSchedulerTest extends BaseTest void testRemovingNoLongerNeededJobsDuringSetupSchedules() throws SchedulerException { QInstance qInstance = QContext.getQInstance(); - QScheduleManager.defineDefaultSchedulableTypesInInstance(qInstance); QuartzTestUtils.setupInstanceForQuartzTests(); //////////////////////////// @@ -167,7 +166,7 @@ class QuartzSchedulerTest extends BaseTest qInstance.addProcess(test1); qInstance.addProcess(test2); - SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.PROCESS.getId()); + SchedulableType schedulableType = qInstance.getSchedulableType(ScheduledJobType.PROCESS.name()); QuartzScheduler quartzScheduler = QuartzScheduler.initInstance(qInstance, QuartzTestUtils.QUARTZ_SCHEDULER_NAME, QuartzTestUtils.getQuartzProperties(), () -> QContext.getQSession()); quartzScheduler.start(); From c9f921c148eaf2459c2947e41e0f745b4cc4cda3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 20:04:43 -0500 Subject: [PATCH 66/72] CE-936 add post-actions on scheduledJobParams table, to reschedule jobs --- .../ScheduledJobsMetaDataProvider.java | 7 + .../ScheduledJobParameterTableCustomizer.java | 215 ++++++++++ .../ScheduledJobTableCustomizer.java | 44 +- .../ScheduledJobTableCustomizerTest.java | 383 ++++++++++++++++++ 4 files changed, 639 insertions(+), 10 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobParameterTableCustomizer.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java index 2e0036ae..f814ff88 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/ScheduledJobsMetaDataProvider.java @@ -45,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers.ScheduledJobParameterTableCustomizer; import com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers.ScheduledJobTableCustomizer; @@ -205,6 +206,12 @@ public class ScheduledJobsMetaDataProvider .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "scheduledJobId", "key", "value"))) .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + QCodeReference customizerReference = new QCodeReference(ScheduledJobParameterTableCustomizer.class); + tableMetaData.withCustomizer(TableCustomizers.POST_INSERT_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, customizerReference); + tableMetaData.withCustomizer(TableCustomizers.POST_DELETE_RECORD, customizerReference); + tableMetaData.withExposedJoin(new ExposedJoin() .withJoinTable(ScheduledJob.TABLE_NAME) .withJoinPath(List.of(JOB_PARAMETER_JOIN_NAME)) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobParameterTableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobParameterTableCustomizer.java new file mode 100644 index 00000000..798efdba --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobParameterTableCustomizer.java @@ -0,0 +1,215 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers; + + +import java.time.Instant; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobParameter; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ScheduledJobParameterTableCustomizer implements TableCustomizerInterface +{ + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postInsert(InsertInput insertInput, List records) throws QException + { + /////////////////////////////////////////////////////////////////////////////////////// + // if we're in this insert as a result of an insert (or update) on a different table // + // (e.g., under a manageAssociations call), then return with noop - assume that the // + // parent table's customizer will do what needed to be done. // + /////////////////////////////////////////////////////////////////////////////////////// + if(!isThisAnActionDirectlyOnThisTable()) + { + return (records); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else - this was an action directly on this table - so bump all of the parent records, to get them rescheduled // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + bumpParentRecords(records, Optional.empty()); + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void bumpParentRecords(List records, Optional> oldRecordList) throws QException + { + try + { + /////////////////////////////////////////////////////////////////////////////////////////// + // (listing) hash up the records by scheduledJobId - we'll use this to have a set of the // + // job ids, and in case we need to add warnings to them later // + /////////////////////////////////////////////////////////////////////////////////////////// + ListingHash recordsByJobId = new ListingHash<>(); + for(QRecord record : records) + { + recordsByJobId.add(record.getValueInteger("scheduledJobId"), record); + } + + Set scheduledJobIds = new HashSet<>(recordsByJobId.keySet()); + + //////////////////////////////////////////////////////////////////////////////// + // if we have an old record list (e.g., is an edit), add any job ids that are // + // in those too, e.g., in case moving a param from one job to another... // + // note, we won't line these up for doing a proper warning on these... // + //////////////////////////////////////////////////////////////////////////////// + if(oldRecordList.isPresent()) + { + for(QRecord oldRecord : oldRecordList.get()) + { + scheduledJobIds.add(oldRecord.getValueInteger("scheduledJobId")); + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // update the modify date on the scheduled jobs - to get their post-actions to run, to reschedule // + //////////////////////////////////////////////////////////////////////////////////////////////////// + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(ScheduledJob.TABLE_NAME); + updateInput.setRecords(scheduledJobIds.stream() + .map(id -> new QRecord().withValue("id", id).withValue("modifyDate", Instant.now())) + .toList()); + UpdateOutput updateOutput = new UpdateAction().execute(updateInput); + + //////////////////////////////////////////////////////////////////////////////////////// + // look for warnings on those jobs - and propagate them to the params we just stored. // + //////////////////////////////////////////////////////////////////////////////////////// + for(QRecord updatedScheduledJob : updateOutput.getRecords()) + { + if(CollectionUtils.nullSafeHasContents(updatedScheduledJob.getWarnings())) + { + for(QRecord paramToWarn : CollectionUtils.nonNullList(recordsByJobId.get(updatedScheduledJob.getValueInteger("id")))) + { + paramToWarn.setWarnings(updatedScheduledJob.getWarnings()); + } + } + } + } + catch(Exception e) + { + LOG.warn("Error in scheduledJobParameter post-crud", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postUpdate(UpdateInput updateInput, List records, Optional> oldRecordList) throws QException + { + ///////////////////////////////////////////////////////////////////////////// + // if we're in this update as a result of an update on a different table // + // (e.g., under a manageAssociations call), then return with noop - assume // + // that the parent table's customizer will do what needed to be done. // + ///////////////////////////////////////////////////////////////////////////// + if(!isThisAnActionDirectlyOnThisTable()) + { + return (records); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else - this was an action directly on this table - so bump all of the parent records, to get them rescheduled // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + bumpParentRecords(records, oldRecordList); + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List postDelete(DeleteInput deleteInput, List records) throws QException + { + ///////////////////////////////////////////////////////////////////////////// + // if we're in this update as a result of an update on a different table // + // (e.g., under a manageAssociations call), then return with noop - assume // + // that the parent table's customizer will do what needed to be done. // + ///////////////////////////////////////////////////////////////////////////// + if(!isThisAnActionDirectlyOnThisTable()) + { + return (records); + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else - this was an action directly on this table - so bump all of the parent records, to get them rescheduled // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + bumpParentRecords(records, Optional.empty()); + + return (records); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean isThisAnActionDirectlyOnThisTable() + { + Optional firstActionInStack = QContext.getFirstActionInStack(); + if(firstActionInStack.isPresent()) + { + if(firstActionInStack.get() instanceof AbstractTableActionInput tableActionInput) + { + if(!ScheduledJobParameter.TABLE_NAME.equals(tableActionInput.getTableName())) + { + return (false); + } + } + } + return (true); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java index f3cc54cf..7203f500 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizer.java @@ -22,10 +22,12 @@ package com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers; -import java.io.Serializable; +import java.text.ParseException; +import java.util.Collections; import java.util.List; import java.util.ListIterator; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; @@ -47,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.quartz.CronScheduleBuilder; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -62,7 +65,7 @@ public class ScheduledJobTableCustomizer implements TableCustomizerInterface @Override public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException { - validateConditionalFields(records); + validateConditionalFields(records, Collections.emptyMap()); return (records); } @@ -86,7 +89,9 @@ public class ScheduledJobTableCustomizer implements TableCustomizerInterface @Override public List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException { - validateConditionalFields(records); + Map freshOldRecordsWithAssociationsMap = CollectionUtils.recordsToMap(freshlyQueryForRecordsWithAssociations(oldRecordList.get()), "id", Integer.class); + + validateConditionalFields(records, freshOldRecordsWithAssociationsMap); if(isPreview || oldRecordList.isEmpty()) { @@ -96,8 +101,7 @@ public class ScheduledJobTableCustomizer implements TableCustomizerInterface /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // refresh the old-records w/ versions that have associations - so we can use those in the post-update to property unschedule things // /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - Map freshOldRecordsWithAssociationsMap = CollectionUtils.recordsToMap(freshlyQueryForRecordsWithAssociations(oldRecordList.get()), "id"); - ListIterator iterator = oldRecordList.get().listIterator(); + ListIterator iterator = oldRecordList.get().listIterator(); while(iterator.hasNext()) { QRecord record = iterator.next(); @@ -116,20 +120,40 @@ public class ScheduledJobTableCustomizer implements TableCustomizerInterface /******************************************************************************* ** *******************************************************************************/ - private static void validateConditionalFields(List records) + private static void validateConditionalFields(List records, Map freshOldRecordsWithAssociationsMap) { + QRecord blankRecord = new QRecord(); for(QRecord record : records) { - if(StringUtils.hasContent(record.getValueString("cronExpression"))) + QRecord oldRecord = Objects.requireNonNullElse(freshOldRecordsWithAssociationsMap.get(record.getValueInteger("id")), blankRecord); + String cronExpression = record.getValues().containsKey("cronExpression") ? record.getValueString("cronExpression") : oldRecord.getValueString("cronExpression"); + String cronTimeZoneId = record.getValues().containsKey("cronTimeZoneId") ? record.getValueString("cronTimeZoneId") : oldRecord.getValueString("cronTimeZoneId"); + String repeatSeconds = record.getValues().containsKey("repeatSeconds") ? record.getValueString("repeatSeconds") : oldRecord.getValueString("repeatSeconds"); + + if(StringUtils.hasContent(cronExpression)) { - if(!StringUtils.hasContent(record.getValueString("cronTimeZoneId"))) + if(StringUtils.hasContent(repeatSeconds)) { - record.addError(new BadInputStatusMessage("If a Cron Expression is given, then a Cron Time Zone Id is required.")); + record.addError(new BadInputStatusMessage("Cron Expression and Repeat Seconds may not both be given.")); + } + + try + { + CronScheduleBuilder.cronScheduleNonvalidatedExpression(cronExpression); + } + catch(ParseException e) + { + record.addError(new BadInputStatusMessage("Cron Expression [" + cronExpression + "] is not valid: " + e.getMessage())); + } + + if(!StringUtils.hasContent(cronTimeZoneId)) + { + record.addError(new BadInputStatusMessage("If a Cron Expression is given, then a Cron Time Zone is required.")); } } else { - if(!StringUtils.hasContent(record.getValueString("repeatSeconds"))) + if(!StringUtils.hasContent(repeatSeconds)) { record.addError(new BadInputStatusMessage("Either Cron Expression or Repeat Seconds must be given.")); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java new file mode 100644 index 00000000..7e926fe1 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/scheduledjobs/customizers/ScheduledJobTableCustomizerTest.java @@ -0,0 +1,383 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.scheduledjobs.customizers; + + +import java.time.Instant; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJob; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobParameter; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobType; +import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobsMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzJobAndTriggerWrapper; +import com.kingsrook.qqq.backend.core.scheduler.quartz.QuartzTestUtils; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; +import org.assertj.core.api.AbstractStringAssert; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.quartz.CronTrigger; +import org.quartz.SchedulerException; +import org.quartz.SimpleTrigger; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for ScheduledJobTableCustomizer + *******************************************************************************/ +class ScheduledJobTableCustomizerTest extends BaseTest +{ + private static final String GOOD_CRON = "0 * * * * ?"; + private static final String GOOD_CRON_2 = "* * * * * ?"; + private static final String BAD_CRON = "* * * * * *"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws QException + { + QInstance qInstance = QContext.getQInstance(); + QuartzTestUtils.setupInstanceForQuartzTests(); + + QSession qSession = QContext.getQSession(); + QScheduleManager qScheduleManager = QScheduleManager.initInstance(qInstance, () -> qSession); + qScheduleManager.start(); + + new ScheduledJobsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QuartzTestUtils.afterEach(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPreInsertAssertValidationErrors() throws QException + { + UnsafeFunction, QRecord, QException> tryToInsert = consumer -> + { + ScheduledJob scheduledJob = new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true); + consumer.accept(scheduledJob); + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(scheduledJob)); + return (insertOutput.getRecords().get(0)); + }; + + ///////////////////////////////////////////////////////// + // lambdas to run a test and assert about no of errors // + ///////////////////////////////////////////////////////// + Function> assertOneErrorExtractingMessage = qRecord -> assertThat(qRecord.getErrors()).hasSize(1).first().extracting("message").asString(); + Consumer assertNoErrors = qRecord -> assertThat(qRecord.getErrors()).hasSize(0); + + assertOneErrorExtractingMessage.apply(tryToInsert.apply(sj -> sj.setId(null))) + .contains("Either Cron Expression or Repeat Seconds must be given"); + + assertOneErrorExtractingMessage.apply(tryToInsert.apply(sj -> sj.withRepeatSeconds(1).withCronExpression(GOOD_CRON).withCronTimeZoneId("UTC"))) + .contains("Cron Expression and Repeat Seconds may not both be given"); + + assertOneErrorExtractingMessage.apply(tryToInsert.apply(sj -> sj.withRepeatSeconds(null).withCronExpression(GOOD_CRON))) + .contains("If a Cron Expression is given, then a Cron Time Zone is required"); + + assertOneErrorExtractingMessage.apply(tryToInsert.apply(sj -> sj.withRepeatSeconds(null).withCronExpression(BAD_CRON).withCronTimeZoneId("UTC"))) + .contains("Support for specifying both a day-of-week AND a day-of-month parameter is not implemented"); + + /////////////////// + // success cases // + /////////////////// + assertNoErrors.accept(tryToInsert.apply(sj -> sj.withCronExpression(GOOD_CRON).withCronTimeZoneId("UTC"))); + assertNoErrors.accept(tryToInsert.apply(sj -> sj.withRepeatSeconds(1))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostInsertActionSchedulesJob() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of( + new ScheduledJobParameter().withKey("processName").withValue(TestUtils.PROCESS_NAME_BASEPULL))))); + + assertTrue(CollectionUtils.nullSafeIsEmpty(insertOutput.getRecords().get(0).getErrors())); + assertTrue(CollectionUtils.nullSafeIsEmpty(insertOutput.getRecords().get(0).getWarnings())); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostInsertActionIssuesWarnings() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + InsertOutput insertOutput = new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of( + new ScheduledJobParameter().withKey("processName").withValue("notAProcess"))))); + + assertTrue(CollectionUtils.nullSafeIsEmpty(insertOutput.getRecords().get(0).getErrors())); + assertThat(insertOutput.getRecords().get(0).getWarnings()) + .hasSize(1).first().extracting("message").asString() + .contains("Error scheduling job: Unrecognized processName"); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPreUpdateAssertValidationErrors() throws QException + { + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1))); + + UnsafeFunction, QRecord, QException> tryToUpdate = consumer -> + { + QRecord record = new QRecord().withValue("id", 1); + consumer.accept(record); + UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(ScheduledJob.TABLE_NAME).withRecord(record)); + return (updateOutput.getRecords().get(0)); + }; + + ///////////////////////////////////////////////////////// + // lambdas to run a test and assert about no of errors // + ///////////////////////////////////////////////////////// + Function> assertOneErrorExtractingMessage = qRecord -> assertThat(qRecord.getErrors()).hasSize(1).first().extracting("message").asString(); + Consumer assertNoErrors = qRecord -> assertThat(qRecord.getErrors()).hasSize(0); + + assertOneErrorExtractingMessage.apply(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null))) + .contains("Either Cron Expression or Repeat Seconds must be given"); + + assertOneErrorExtractingMessage.apply(tryToUpdate.apply(r -> r.withValue("cronExpression", GOOD_CRON).withValue("cronTimeZoneId", "UTC"))) + .contains("Cron Expression and Repeat Seconds may not both be given"); + + assertOneErrorExtractingMessage.apply(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", GOOD_CRON))) + .contains("If a Cron Expression is given, then a Cron Time Zone is required"); + + assertOneErrorExtractingMessage.apply(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", BAD_CRON).withValue("cronTimeZoneId", "UTC"))) + .contains("Support for specifying both a day-of-week AND a day-of-month parameter is not implemented"); + + /////////////////// + // success cases // + /////////////////// + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("modifyDate", Instant.now()))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", GOOD_CRON).withValue("cronTimeZoneId", "UTC"))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", GOOD_CRON_2).withValue("cronTimeZoneId", "UTC"))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", null).withValue("cronExpression", GOOD_CRON).withValue("cronTimeZoneId", "America/Chicago"))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", 1).withValue("cronExpression", null).withValue("cronTimeZoneId", null))); + assertNoErrors.accept(tryToUpdate.apply(r -> r.withValue("repeatSeconds", 2))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostUpdateActionReSchedulesJob() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + ////////////////////////////////////////////////// + // do an insert - this will originally schedule // + ////////////////////////////////////////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of(new ScheduledJobParameter().withKey("processName").withValue(TestUtils.PROCESS_NAME_BASEPULL))))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + assertThat(wrappers.get(0).trigger()).isInstanceOf(SimpleTrigger.class); + + ////////////////////////////////////// + // now do an update, to re-schedule // + ////////////////////////////////////// + new UpdateAction().execute(new UpdateInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withId(1) + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(null) + .withCronExpression(GOOD_CRON) + .withCronTimeZoneId("UTC"))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + assertThat(wrappers.get(0).trigger()).isInstanceOf(CronTrigger.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostUpdateActionIssuesWarnings() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + ////////////////////////////////////////////////// + // do an insert - this will originally schedule // + ////////////////////////////////////////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of(new ScheduledJobParameter().withKey("processName").withValue(TestUtils.PROCESS_NAME_BASEPULL))))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + assertThat(wrappers.get(0).trigger()).isInstanceOf(SimpleTrigger.class); + + ////////////////////////////////////// + // now do an update, to re-schedule // + ////////////////////////////////////// + UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withId(1) + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(null) + .withCronExpression(GOOD_CRON) + .withCronTimeZoneId("UTC") + .withJobParameters(List.of(new ScheduledJobParameter().withKey("process").withValue("not"))))); + + assertTrue(CollectionUtils.nullSafeIsEmpty(updateOutput.getRecords().get(0).getErrors())); + assertThat(updateOutput.getRecords().get(0).getWarnings()) + .hasSize(1).first().extracting("message").asString() + .contains("Missing scheduledJobParameter with key [processName]"); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPostDeleteUnschedules() throws QException, SchedulerException + { + List wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + + ////////////////////////////////////////////////// + // do an insert - this will originally schedule // + ////////////////////////////////////////////////// + new InsertAction().execute(new InsertInput(ScheduledJob.TABLE_NAME).withRecordEntity(new ScheduledJob() + .withLabel("Test") + .withSchedulerName(QuartzTestUtils.QUARTZ_SCHEDULER_NAME) + .withType(ScheduledJobType.PROCESS.name()) + .withIsActive(true) + .withRepeatSeconds(1) + .withJobParameters(List.of(new ScheduledJobParameter().withKey("processName").withValue(TestUtils.PROCESS_NAME_BASEPULL))))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(1, wrappers.size()); + + //////////////////////////////////// + // now do a delete, to unschedule // + //////////////////////////////////// + new DeleteAction().execute(new DeleteInput(ScheduledJob.TABLE_NAME).withPrimaryKeys(List.of(1))); + + wrappers = QuartzTestUtils.queryQuartz(); + assertEquals(0, wrappers.size()); + } + +} \ No newline at end of file From 02d068dad7b9af95c1fb2b288b49e125f7016c05 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 20:05:00 -0500 Subject: [PATCH 67/72] CE-936 Quartz test updates --- .../scheduler/quartz/QuartzSchedulerTest.java | 22 +------------ .../scheduler/quartz/QuartzTestUtils.java | 32 +++++++++++++++++++ .../processes/QuartzJobsProcessTest.java | 23 +------------ 3 files changed, 34 insertions(+), 43 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java index f83972c2..f0b58b0d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java @@ -62,27 +62,7 @@ class QuartzSchedulerTest extends BaseTest @AfterEach void afterEach() { - try - { - QScheduleManager.getInstance().unInit(); - } - catch(IllegalStateException ise) - { - ///////////////////////////////////////////////////////////////// - // ok, might just mean that this test didn't init the instance // - ///////////////////////////////////////////////////////////////// - } - - try - { - QuartzScheduler.getInstance().unInit(); - } - catch(IllegalStateException ise) - { - ///////////////////////////////////////////////////////////////// - // ok, might just mean that this test didn't init the instance // - ///////////////////////////////////////////////////////////////// - } + QuartzTestUtils.afterEach(); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java index 3e952b50..a82d163c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzTestUtils.java @@ -27,6 +27,7 @@ import java.util.Properties; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData; +import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; import org.quartz.SchedulerException; @@ -102,4 +103,35 @@ public class QuartzTestUtils { return QuartzScheduler.getInstance().queryQuartz(); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void afterEach() + { + try + { + QScheduleManager.getInstance().stop(); + QScheduleManager.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + + try + { + QuartzScheduler.getInstance().unInit(); + } + catch(IllegalStateException ise) + { + ///////////////////////////////////////////////////////////////// + // ok, might just mean that this test didn't init the instance // + ///////////////////////////////////////////////////////////////// + } + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java index 5ce1c175..023d6765 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java @@ -92,28 +92,7 @@ class QuartzJobsProcessTest extends BaseTest @AfterEach void afterEach() { - try - { - QScheduleManager.getInstance().stop(); - QScheduleManager.getInstance().unInit(); - } - catch(IllegalStateException ise) - { - ///////////////////////////////////////////////////////////////// - // ok, might just mean that this test didn't init the instance // - ///////////////////////////////////////////////////////////////// - } - - try - { - QuartzScheduler.getInstance().unInit(); - } - catch(IllegalStateException ise) - { - ///////////////////////////////////////////////////////////////// - // ok, might just mean that this test didn't init the instance // - ///////////////////////////////////////////////////////////////// - } + QuartzTestUtils.afterEach(); } From 7e6a3c528f79c8f183c4f45341a30e8b8bc791c8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Mar 2024 20:12:36 -0500 Subject: [PATCH 68/72] CE-936 Update tests --- .../backend/core/scheduler/quartz/QuartzSchedulerTest.java | 2 +- .../scheduler/quartz/processes/QuartzJobsProcessTest.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java index f0b58b0d..46f5c8cc 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzSchedulerTest.java @@ -100,7 +100,7 @@ class QuartzSchedulerTest extends BaseTest ////////////////////////////////////////////////// // give a moment for the job to run a few times // ////////////////////////////////////////////////// - SleepUtils.sleep(50, TimeUnit.MILLISECONDS); + SleepUtils.sleep(150, TimeUnit.MILLISECONDS); qScheduleManager.stopAsync(); System.out.println("Ran: " + BasicStep.counter + " times"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java index 023d6765..f2bf6067 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/quartz/processes/QuartzJobsProcessTest.java @@ -67,7 +67,7 @@ class QuartzJobsProcessTest extends BaseTest { QInstance qInstance = QContext.getQInstance(); qInstance.addTable(new QTableMetaData() - .withName("quartzTriggers") + .withName("quartzJobDetails") .withBackendName(TestUtils.MEMORY_BACKEND_NAME) .withPrimaryKeyField("id") .withField(new QFieldMetaData("id", QFieldType.LONG))); @@ -162,7 +162,7 @@ class QuartzJobsProcessTest extends BaseTest // pause just one // //////////////////// List quartzJobAndTriggerWrappers = QuartzTestUtils.queryQuartz(); - new InsertAction().execute(new InsertInput("quartzTriggers").withRecord(new QRecord() + new InsertAction().execute(new InsertInput("quartzJobDetails").withRecord(new QRecord() .withValue("jobName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()) .withValue("jobGroup", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getGroup()) )); @@ -210,7 +210,7 @@ class QuartzJobsProcessTest extends BaseTest // pause just one // //////////////////// List quartzJobAndTriggerWrappers = QuartzTestUtils.queryQuartz(); - new InsertAction().execute(new InsertInput("quartzTriggers").withRecord(new QRecord() + new InsertAction().execute(new InsertInput("quartzJobDetails").withRecord(new QRecord() .withValue("jobName", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getName()) .withValue("jobGroup", quartzJobAndTriggerWrappers.get(0).jobDetail().getKey().getGroup()) )); From 0d6538593bb84ae86a4692799fb836199e7a867d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 20 Mar 2024 13:49:33 -0500 Subject: [PATCH 69/72] CE-936 Add option to log the running of quartz jobs --- .../core/scheduler/quartz/QuartzJobRunner.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java index 64831ac9..2972f8ac 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java @@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; +import org.apache.logging.log4j.Level; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; @@ -43,6 +44,7 @@ public class QuartzJobRunner implements Job { private static final QLogger LOG = QLogger.getLogger(QuartzJobRunner.class); + private static Level logLevel = null; /******************************************************************************* @@ -62,6 +64,10 @@ public class QuartzJobRunner implements Job Map params = (Map) context.getJobDetail().getJobDataMap().get("params"); SchedulableRunner schedulableRunner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + if(logLevel != null) + { + LOG.log(logLevel, "Running QuartzJob", null, logPair("type", schedulableType.getName()), logPair("name", context.getJobDetail().getKey().getName()), logPair("params", params)); + } schedulableRunner.run(params); } catch(Exception e) @@ -74,4 +80,13 @@ public class QuartzJobRunner implements Job } } + + /******************************************************************************* + ** + *******************************************************************************/ + public static void setLogLevel(Level level) + { + logLevel = level; + } + } From 30ee8ce3bf459bef99ebff57faf9054cff96bff1 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 20 Mar 2024 16:13:27 -0500 Subject: [PATCH 70/72] Update quartz job runner to log when finished (and to include same pairs in exception) --- .../scheduler/quartz/QuartzJobRunner.java | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java index 2972f8ac..59a1eebc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java @@ -47,6 +47,7 @@ public class QuartzJobRunner implements Job private static Level logLevel = null; + /******************************************************************************* ** *******************************************************************************/ @@ -54,25 +55,38 @@ public class QuartzJobRunner implements Job public void execute(JobExecutionContext context) throws JobExecutionException { CapturedContext capturedContext = QContext.capture(); + + String name = null; + SchedulableType schedulableType = null; + Map params = null; try { + name = context.getJobDetail().getKey().getName(); + QuartzScheduler quartzScheduler = QuartzScheduler.getInstance(); QInstance qInstance = quartzScheduler.getQInstance(); QContext.init(qInstance, quartzScheduler.getSessionSupplier().get()); - SchedulableType schedulableType = qInstance.getSchedulableType(context.getJobDetail().getJobDataMap().getString("type")); - Map params = (Map) context.getJobDetail().getJobDataMap().get("params"); + schedulableType = qInstance.getSchedulableType(context.getJobDetail().getJobDataMap().getString("type")); + params = (Map) context.getJobDetail().getJobDataMap().get("params"); SchedulableRunner schedulableRunner = QCodeLoader.getAdHoc(SchedulableRunner.class, schedulableType.getRunner()); + if(logLevel != null) { - LOG.log(logLevel, "Running QuartzJob", null, logPair("type", schedulableType.getName()), logPair("name", context.getJobDetail().getKey().getName()), logPair("params", params)); + LOG.log(logLevel, "Running QuartzJob", null, logPair("name", name), logPair("type", schedulableType.getName()), logPair("params", params)); } + schedulableRunner.run(params); + + if(logLevel != null) + { + LOG.log(logLevel, "Finished QuartzJob", null, logPair("name", name), logPair("type", schedulableType.getName()), logPair("params", params)); + } } catch(Exception e) { - LOG.warn("Error running QuartzJob", e, logPair("jobContext", context)); + LOG.warn("Error running QuartzJob", e, logPair("name", name), logPair("type", schedulableType == null ? null : schedulableType.getName()), logPair("params", params)); } finally { @@ -81,6 +95,7 @@ public class QuartzJobRunner implements Job } + /******************************************************************************* ** *******************************************************************************/ From 40e8a85977b597df6a40aa6264ffc19d2afe6acc Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 20 Mar 2024 16:13:39 -0500 Subject: [PATCH 71/72] Add ScheduledJobs doc --- docs/index.adoc | 12 ++- docs/misc/ScheduledJobs.adoc | 141 +++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 docs/misc/ScheduledJobs.adoc diff --git a/docs/index.adoc b/docs/index.adoc index 1ab41a32..d777f0cc 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -26,6 +26,16 @@ include::metaData/Reports.adoc[leveloffset=+1] include::metaData/Icons.adoc[leveloffset=+1] include::metaData/PermissionRules.adoc[leveloffset=+1] +== Services + +include::misc/ScheduledJobs.adoc[leveloffset=+1] + +=== Web server (Javalin) +#todo# + +=== API server (OpenAPI) +#todo# + == Custom Application Code include::misc/QContext.adoc[leveloffset=+1] include::misc/QRecords.adoc[leveloffset=+1] @@ -63,4 +73,4 @@ include::implementations/TableSync.adoc[leveloffset=+1] // later... include::actions/RenderTemplateAction.adoc[leveloffset=+1] == QQQ Utility Classes -include::utilities/RecordLookupHelper.adoc[leveloffset=+1] \ No newline at end of file +include::utilities/RecordLookupHelper.adoc[leveloffset=+1] diff --git a/docs/misc/ScheduledJobs.adoc b/docs/misc/ScheduledJobs.adoc new file mode 100644 index 00000000..b4a88a61 --- /dev/null +++ b/docs/misc/ScheduledJobs.adoc @@ -0,0 +1,141 @@ +== Schedulers and Scheduled Jobs +include::../variables.adoc[] + +QQQ has the ability to automatically run various types of jobs on schedules, +either defined in your instance's meta-data, +or optionally via data in your application, in a `scheduledJob` table. + +=== Schedulers and QSchedulerMetaData +2 types of schedulers are included in QQQ by default (though an application can define its own schedulers): + +* `SimpleScheduler` - is (as its name suggests) a simple class which uses java's `ScheduledExecutorService` +to run jobs on repeating intervals. +** Cannot run cron schedules - only repeating intervals. +** If multiple servers are running, each will potentially run the same job concurrently +** Has no configurations, e.g., to limit the number of threads. + +* `QuartzScheduler` - uses the 3rd party https://www.quartz-scheduler.org/[Quartz Scheduler] library to provide +a much more capable, though admittedly more complex, scheduling solution. +** Can run both cron schedules and repeating intervals. +** By default, will not allow concurrent executions of the same job. +** Supports multiple configurations, e.g., to limit the number of threads. + +An application can define its own scheduler by providing a class which implements the `QSchedulerInterface`. + +A `QInstance` can work with 0 or more schedulers, as defined by adding `QSchedulerMetaData` objects +to the instance. + +This meta-data class is `abstract`, and is extended by the 2 built-in schedulers +(e.g., `SimpleSchedulerMetaData` and `QuartzSchedulerMetaData`). As such, +these concrete subclasses are what you need to instantiate and add to your instance. + +To configure a QuartzScheduler, you can add a `Properties` object to the `QuartzSchedulerMetaData` object. +See https://www.quartz-scheduler.org/documentation/[Quartz's documentation] for available configuration properties. + +[source,java] +.Defining SchedulerMetaData +---- +qInstance.addScheduler(new SimpleSchedulerMetaData().withName("mySimpleScheduler")); + +qInstance.addScheduler(new QuartzSchedulerMetaData() + .withName("myQuartzScheduler") + .withProperties(myQuartzProperties); +---- + +=== SchedulableTypes +The types of jobs which can be scheduled in a QQQ application are defined in the `QInstance` by +instances of the `SchedulableType` meta-data class. +These objects contain a name, along with a `QCodeReference` to the `runner`, +which must be a class that implements the `SchedulableRunner` interface. + +By default, (in the `QInstanceEnricher`), QQQ will make 3 `SchedulableType` options available: + +* `PROCESS` - Any Process defined in the `QInstance` can be scheduled. +* `QUEUE_PROCESSOR` - A Queue defined in the `QInstance`, which requires polling (e.g., SQS), can be scheduled. +* `TABLE_AUTOMATIONS` - A Table in the `QInstance`, with `AutomationDetails` referring to an +AutomationProvider which requires polling, can be scheduled. + +If an application only wants to use a subset of these `SchedulableType` options, +or to add custom `SchedulableType` options, +the `QInstance` will need to have 1 or more `SchedulableType` objects in it before the `QInstanceEnricher` runs. + +=== User-defined Scheduled Jobs +To allow users to schedule jobs (rather than using programmer-defined schedules (in meta-data)), +you can add a set of tables to your `QInstance`, using the `ScheduledJobsMetaDataProvider` class: + +[source,java] +.Adding the ScheduledJob tables and related meta-data to a QInstance +---- +new ScheduledJobsMetaDataProvider().defineAll( + qInstance, backendName, table -> tableEnricher(table)); +---- + +This meta-data provider adds a "scheduledJob" and "scheduledJobParameter" table, along with +some PossibleValueSources. +These tables include post-action customizers, which manage (re-, un-) scheduling jobs based on +changes made to records in this these tables. + +Also, when `QScheduleManager` is started, it will query these tables,and will schedule jobs as defined therein. + +_You can use a mix of user-defined and meta-data defined scheduled jobs in your instance. +However, if a ScheduledJob record references a process, queue, or table automation with a +meta-data defined schedule, the ScheduledJob will NOT be started by ScheduleManager -- +rather, the meta-data definition will "win"._ + +[source,sql] +.Example of inserting scheduled jobs records directly into an SQL backend +---- +-- A process: +INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES + ('myProcess', 'QuartzScheduler', null, null, 300, 'PROCESS', 1); +INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES + ((SELECT id FROM scheduled_job WHERE label = 'myProcess'), 'processName', 'myProcess'); + +-- A table's insert & update automations: +INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES + ('myTable.PENDING_INSERT_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1), + ('myTable.PENDING_UPDATE_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1); +INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES + ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'tableName', 'myTable'), + ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'automationStatus', 'PENDING_INSERT_AUTOMATIONS'), + ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'tableName', 'myTable'), + ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'automationStatus', 'PENDING_UPDATE_AUTOMATIONS'); + +-- A queue processor: +INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES + ('mySqsQueue', 'QuartzScheduler', null, null, 60, 'QUEUE_PROCESSOR', 1); +INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES + ((SELECT id FROM scheduled_job WHERE label = 'mySqsQueue'), 'queueName', 'mySqsQueue'); +---- + +=== Running Scheduled Jobs +In a server running QQQ, if you wish to start running scheduled jobs, you need to initialize +the `QScheduleManger` singleton class, then call its `start()` method. + +Note that internally, this class will check for a system property of `qqq.scheduleManager.enabled` +or an environment variable of `QQQ_SCHEDULE_MANAGER_ENABLED`, and if the first value found is +`"false"`, then the scheduler will not actually run its jobs (but, in the case of the `QuartzSchdeuler`, +it will be available for managing scheduled jobs). + +The method `QScheduleManager.initInstance` requires 2 parameters: Your `QInstance`, and a +`Supplier` lambda, which returns the session that will be used for scheduled jobs when they +are executed. + +[source,java] +.Starting the Schedule Manager service +---- +QScheduleManager.initInstance(qInstance, () -> systemUserSession).start(); +---- + +=== Examples +[source,java] +.Attach a schedule in meta-data to a Process +---- +QProcessMetaData myProcess = new QProcessMetaData() + // ... + .withSchedule(new QScheduleMetaData() + .withSchedulerName("myScheduler") + .withDescription("Run myProcess every five minutes") + .withRepeatSeconds(300)) +---- + From d7dec3e56047d427432bf03467aea3bca569dbad Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 20 Mar 2024 17:42:05 -0500 Subject: [PATCH 72/72] CE-936 Add @DisallowConcurrentExecution --- .../qqq/backend/core/scheduler/quartz/QuartzJobRunner.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java index 59a1eebc..f635bd9b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/quartz/QuartzJobRunner.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.scheduler.schedulable.SchedulableType; import com.kingsrook.qqq.backend.core.scheduler.schedulable.runner.SchedulableRunner; import org.apache.logging.log4j.Level; +import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; @@ -40,6 +41,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* ** *******************************************************************************/ +@DisallowConcurrentExecution public class QuartzJobRunner implements Job { private static final QLogger LOG = QLogger.getLogger(QuartzJobRunner.class);