diff --git a/pom.xml b/pom.xml index 1de39e7c..de49e349 100644 --- a/pom.xml +++ b/pom.xml @@ -36,6 +36,7 @@ qqq-middleware-picocli qqq-middleware-javalin qqq-middleware-lambda + qqq-utility-lambdas qqq-sample-project @@ -51,8 +52,20 @@ true 0.80 0.95 + none + + + + buildShadedJar + + package + + + + @@ -88,7 +101,7 @@ 3.23.1 test - + diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index 621642eb..686049ca 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -115,6 +115,12 @@ v3-rev20220815-2.0.0 + + com.amazonaws + aws-java-sdk-sqs + 1.12.321 + + org.apache.maven.plugins @@ -175,7 +181,7 @@ - package + ${plugin.shade.phase} shade diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java index 896827f2..18012827 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java @@ -67,7 +67,7 @@ import org.apache.logging.log4j.Logger; ** Runnable for the Polling Automation Provider, that looks for records that ** need automations, and executes them. *******************************************************************************/ -class PollingAutomationRunner implements Runnable +public class PollingAutomationRunner implements Runnable { private static final Logger LOG = LogManager.getLogger(PollingAutomationRunner.class); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/queues/SQSQueuePoller.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/queues/SQSQueuePoller.java new file mode 100644 index 00000000..bb5f9705 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/queues/SQSQueuePoller.java @@ -0,0 +1,167 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.queues; + + +import java.util.function.Supplier; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.DeleteMessageRequest; +import com.amazonaws.services.sqs.model.Message; +import com.amazonaws.services.sqs.model.ReceiveMessageRequest; +import com.amazonaws.services.sqs.model.ReceiveMessageResult; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Class to poll an SQS queue, and run process code for each message found. + *******************************************************************************/ +public class SQSQueuePoller implements Runnable +{ + private static final Logger LOG = LogManager.getLogger(SQSQueuePoller.class); + + /////////////////////////////////////////////// + // todo - move these 2 to a "QBaseRunnable"? // + /////////////////////////////////////////////// + private QInstance qInstance; + private Supplier sessionSupplier; + + private SQSQueueProviderMetaData queueProviderMetaData; + private QQueueMetaData queueMetaData; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run() + { + try + { + BasicAWSCredentials credentials = new BasicAWSCredentials(queueProviderMetaData.getAccessKey(), queueProviderMetaData.getSecretKey()); + final AmazonSQS sqs = AmazonSQSClientBuilder.standard() + .withRegion(queueProviderMetaData.getRegion()) + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .build(); + + String queueUrl = queueProviderMetaData.getBaseURL(); + if(!queueUrl.endsWith("/")) + { + queueUrl += "/"; + } + queueUrl += queueMetaData.getQueueName(); + + while(true) + { + ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(); + receiveMessageRequest.setQueueUrl(queueUrl); + ReceiveMessageResult receiveMessageResult = sqs.receiveMessage(receiveMessageRequest); + if(receiveMessageResult.getMessages().isEmpty()) + { + LOG.debug("0 messages received. Breaking."); + break; + } + LOG.debug(receiveMessageResult.getMessages().size() + " messages received. Processing."); + + for(Message message : receiveMessageResult.getMessages()) + { + String body = message.getBody(); + + RunProcessInput runProcessInput = new RunProcessInput(qInstance); + runProcessInput.setSession(sessionSupplier.get()); + runProcessInput.setProcessName(queueMetaData.getProcessName()); + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + runProcessInput.addValue("body", body); + + RunProcessAction runProcessAction = new RunProcessAction(); + RunProcessOutput runProcessOutput = runProcessAction.execute(runProcessInput); + + ///////////////////////////////// + // todo - what of exceptions?? // + ///////////////////////////////// + + String receiptHandle = message.getReceiptHandle(); + sqs.deleteMessage(new DeleteMessageRequest(queueUrl, receiptHandle)); + } + } + } + catch(Exception e) + { + LOG.warn("Error receiving SQS Message", e); + } + } + + + + /******************************************************************************* + ** Setter for queueProviderMetaData + ** + *******************************************************************************/ + public void setQueueProviderMetaData(SQSQueueProviderMetaData queueProviderMetaData) + { + this.queueProviderMetaData = queueProviderMetaData; + } + + + + /******************************************************************************* + ** Setter for queueMetaData + ** + *******************************************************************************/ + public void setQueueMetaData(QQueueMetaData queueMetaData) + { + this.queueMetaData = queueMetaData; + } + + + + /******************************************************************************* + ** Setter for qInstance + ** + *******************************************************************************/ + public void setQInstance(QInstance qInstance) + { + this.qInstance = qInstance; + } + + + + /******************************************************************************* + ** Setter for sessionSupplier + ** + *******************************************************************************/ + public void setSessionSupplier(Supplier sessionSupplier) + { + this.sessionSupplier = sessionSupplier; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 8f81d232..9fc3859e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -36,6 +36,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; @@ -69,6 +71,9 @@ public class QInstance private Map widgets = new LinkedHashMap<>(); + private Map queueProviders = new LinkedHashMap<>(); + private Map queues = new LinkedHashMap<>(); + // todo - lock down the object (no more changes allowed) after it's been validated? @JsonIgnore @@ -670,4 +675,124 @@ public class QInstance return (this.widgets.get(name)); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addQueueProvider(QQueueProviderMetaData queueProvider) + { + this.addQueueProvider(queueProvider.getName(), queueProvider); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addQueueProvider(String name, QQueueProviderMetaData queueProvider) + { + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add an queueProvider without a name.")); + } + if(this.queueProviders.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second queueProvider with name: " + name)); + } + this.queueProviders.put(name, queueProvider); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QQueueProviderMetaData getQueueProvider(String name) + { + return (this.queueProviders.get(name)); + } + + + + /******************************************************************************* + ** Getter for queueProviders + ** + *******************************************************************************/ + public Map getQueueProviders() + { + return queueProviders; + } + + + + /******************************************************************************* + ** Setter for queueProviders + ** + *******************************************************************************/ + public void setQueueProviders(Map queueProviders) + { + this.queueProviders = queueProviders; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addQueue(QQueueMetaData queue) + { + this.addQueue(queue.getName(), queue); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addQueue(String name, QQueueMetaData queue) + { + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add an queue without a name.")); + } + if(this.queues.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second queue with name: " + name)); + } + this.queues.put(name, queue); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QQueueMetaData getQueue(String name) + { + return (this.queues.get(name)); + } + + + + /******************************************************************************* + ** Getter for queues + ** + *******************************************************************************/ + public Map getQueues() + { + return queues; + } + + + + /******************************************************************************* + ** Setter for queues + ** + *******************************************************************************/ + public void setQueues(Map queues) + { + this.queues = queues; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java index 51d1cda0..30fc50f9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java @@ -22,14 +22,20 @@ package com.kingsrook.qqq.backend.core.model.metadata.automation; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; + + /******************************************************************************* ** Meta-data definition of a qqq service to drive record automations. *******************************************************************************/ public class QAutomationProviderMetaData { - private String name; + private String name; private QAutomationProviderType type; + private QScheduleMetaData schedule; + + /******************************************************************************* ** Getter for name @@ -52,6 +58,7 @@ public class QAutomationProviderMetaData } + /******************************************************************************* ** Fluent setter for name ** @@ -85,6 +92,7 @@ public class QAutomationProviderMetaData } + /******************************************************************************* ** Fluent setter for type ** @@ -95,4 +103,38 @@ public class QAutomationProviderMetaData return (this); } + + + /******************************************************************************* + ** Getter for schedule + ** + *******************************************************************************/ + public QScheduleMetaData getSchedule() + { + return schedule; + } + + + + /******************************************************************************* + ** Setter for schedule + ** + *******************************************************************************/ + public void setSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + } + + + + /******************************************************************************* + ** Fluent setter for schedule + ** + *******************************************************************************/ + public QAutomationProviderMetaData withSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index 182e7496..83435da8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -32,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; /******************************************************************************* @@ -51,6 +52,8 @@ public class QProcessMetaData implements QAppChildMetaData private String parentAppName; private QIcon icon; + private QScheduleMetaData schedule; + /******************************************************************************* @@ -436,4 +439,38 @@ public class QProcessMetaData implements QAppChildMetaData return (this); } + + + /******************************************************************************* + ** Getter for schedule + ** + *******************************************************************************/ + public QScheduleMetaData getSchedule() + { + return schedule; + } + + + + /******************************************************************************* + ** Setter for schedule + ** + *******************************************************************************/ + public void setSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + } + + + + /******************************************************************************* + ** Fluent setter for schedule + ** + *******************************************************************************/ + public QProcessMetaData withSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QQueueMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QQueueMetaData.java new file mode 100644 index 00000000..e0822b0d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QQueueMetaData.java @@ -0,0 +1,216 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.queues; + + +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; + + +/******************************************************************************* + ** MetaData to define a message queue, which must exist within a QueueProvider. + ** + ** The name attribute is a globally unique name within the QInstance + ** The providerName is the connection to the queue system. + ** The queueName uniquely identifies the queue within the context of the provider. + ** The processName is the code that runs for messages found on the queue. + ** The schedule may not be used by all provider types, but defines when the queue is polled. + *******************************************************************************/ +public class QQueueMetaData +{ + private String name; + private String providerName; + private String queueName; + private String processName; + + private QScheduleMetaData schedule; + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Setter for name + ** + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public QQueueMetaData withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for providerName + ** + *******************************************************************************/ + public String getProviderName() + { + return providerName; + } + + + + /******************************************************************************* + ** Setter for providerName + ** + *******************************************************************************/ + public void setProviderName(String providerName) + { + this.providerName = providerName; + } + + + + /******************************************************************************* + ** Fluent setter for providerName + ** + *******************************************************************************/ + public QQueueMetaData withProviderName(String providerName) + { + this.providerName = providerName; + return (this); + } + + + + /******************************************************************************* + ** Getter for queueName + ** + *******************************************************************************/ + public String getQueueName() + { + return queueName; + } + + + + /******************************************************************************* + ** Setter for queueName + ** + *******************************************************************************/ + public void setQueueName(String queueName) + { + this.queueName = queueName; + } + + + + /******************************************************************************* + ** Fluent setter for queueName + ** + *******************************************************************************/ + public QQueueMetaData withQueueName(String queueName) + { + this.queueName = queueName; + return (this); + } + + + + /******************************************************************************* + ** Getter for processName + ** + *******************************************************************************/ + public String getProcessName() + { + return processName; + } + + + + /******************************************************************************* + ** Setter for processName + ** + *******************************************************************************/ + public void setProcessName(String processName) + { + this.processName = processName; + } + + + + /******************************************************************************* + ** Fluent setter for processName + ** + *******************************************************************************/ + public QQueueMetaData withProcessName(String processName) + { + this.processName = processName; + return (this); + } + + + + /******************************************************************************* + ** Getter for schedule + ** + *******************************************************************************/ + public QScheduleMetaData getSchedule() + { + return schedule; + } + + + + /******************************************************************************* + ** Setter for schedule + ** + *******************************************************************************/ + public void setSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + } + + + + /******************************************************************************* + ** Fluent setter for schedule + ** + *******************************************************************************/ + public QQueueMetaData withSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QQueueProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QQueueProviderMetaData.java new file mode 100644 index 00000000..bf0faec6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QQueueProviderMetaData.java @@ -0,0 +1,101 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.queues; + + +/******************************************************************************* + ** Define a provider of queues (e.g., an MQ system, or SQS) + *******************************************************************************/ +public class QQueueProviderMetaData +{ + private String name; + private QueueType type; + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Setter for name + ** + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public QQueueProviderMetaData withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public QueueType getType() + { + return type; + } + + + + /******************************************************************************* + ** Setter for type + ** + *******************************************************************************/ + public void setType(QueueType type) + { + this.type = type; + } + + + + /******************************************************************************* + ** Fluent setter for type + ** + *******************************************************************************/ + public QQueueProviderMetaData withType(QueueType type) + { + this.type = type; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QueueType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QueueType.java new file mode 100644 index 00000000..e0f099f2 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QueueType.java @@ -0,0 +1,34 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.queues; + + +/******************************************************************************* + ** Types of queues supported by QQQ. + *******************************************************************************/ +public enum QueueType +{ + SQS + // todo MQ + // todo? IN_MEMORY + // todo? TABLE +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java new file mode 100644 index 00000000..3825b9ff --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java @@ -0,0 +1,238 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.queues; + + +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; + + +/******************************************************************************* + ** Meta-data for an source of Amazon SQS queues (e.g, an aws account/credential + ** set, with a common base URL). + ** + ** Scheduled can be defined here, to apply to all queues in the provider - or + ** each can supply their own schedule. + *******************************************************************************/ +public class SQSQueueProviderMetaData extends QQueueProviderMetaData +{ + private String accessKey; + private String secretKey; + private String region; + private String baseURL; + + private QScheduleMetaData schedule; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SQSQueueProviderMetaData() + { + super(); + setType(QueueType.SQS); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public SQSQueueProviderMetaData withName(String name) + { + super.withName(name); + return (this); + } + + + + /******************************************************************************* + ** Getter for accessKey + ** + *******************************************************************************/ + public String getAccessKey() + { + return accessKey; + } + + + + /******************************************************************************* + ** Setter for accessKey + ** + *******************************************************************************/ + public void setAccessKey(String accessKey) + { + this.accessKey = accessKey; + } + + + + /******************************************************************************* + ** Fluent setter for accessKey + ** + *******************************************************************************/ + public SQSQueueProviderMetaData withAccessKey(String accessKey) + { + this.accessKey = accessKey; + return (this); + } + + + + /******************************************************************************* + ** Getter for secretKey + ** + *******************************************************************************/ + public String getSecretKey() + { + return secretKey; + } + + + + /******************************************************************************* + ** Setter for secretKey + ** + *******************************************************************************/ + public void setSecretKey(String secretKey) + { + this.secretKey = secretKey; + } + + + + /******************************************************************************* + ** Fluent setter for secretKey + ** + *******************************************************************************/ + public SQSQueueProviderMetaData withSecretKey(String secretKey) + { + this.secretKey = secretKey; + return (this); + } + + + + /******************************************************************************* + ** Getter for region + ** + *******************************************************************************/ + public String getRegion() + { + return region; + } + + + + /******************************************************************************* + ** Setter for region + ** + *******************************************************************************/ + public void setRegion(String region) + { + this.region = region; + } + + + + /******************************************************************************* + ** Fluent setter for region + ** + *******************************************************************************/ + public SQSQueueProviderMetaData withRegion(String region) + { + this.region = region; + return (this); + } + + + + /******************************************************************************* + ** Getter for baseURL + ** + *******************************************************************************/ + public String getBaseURL() + { + return baseURL; + } + + + + /******************************************************************************* + ** Setter for baseURL + ** + *******************************************************************************/ + public void setBaseURL(String baseURL) + { + this.baseURL = baseURL; + } + + + + /******************************************************************************* + ** Fluent setter for baseURL + ** + *******************************************************************************/ + public SQSQueueProviderMetaData withBaseURL(String baseURL) + { + this.baseURL = baseURL; + return (this); + } + + + + /******************************************************************************* + ** Getter for schedule + ** + *******************************************************************************/ + public QScheduleMetaData getSchedule() + { + return schedule; + } + + + + /******************************************************************************* + ** Setter for schedule + ** + *******************************************************************************/ + public void setSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + } + + + + /******************************************************************************* + ** Fluent setter for schedule + ** + *******************************************************************************/ + public SQSQueueProviderMetaData withSchedule(QScheduleMetaData schedule) + { + this.schedule = schedule; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java new file mode 100644 index 00000000..f5a7dba3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java @@ -0,0 +1,177 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.scheduleing; + + +/******************************************************************************* + ** Meta-data to define scheduled actions within QQQ. + ** + ** Initially, only supports repeating jobs, either on a given # of seconds or millis. + ** Can also specify an initialDelay - e.g., to avoid all jobs starting up at the + ** same moment. + ** + ** In the future we most likely would want to allow cron strings to be added here. + *******************************************************************************/ +public class QScheduleMetaData +{ + private Integer repeatSeconds; + private Integer repeatMillis; + private Integer initialDelaySeconds; + private Integer initialDelayMillis; + + + + /******************************************************************************* + ** Getter for repeatSeconds + ** + *******************************************************************************/ + public Integer getRepeatSeconds() + { + return repeatSeconds; + } + + + + /******************************************************************************* + ** Setter for repeatSeconds + ** + *******************************************************************************/ + public void setRepeatSeconds(Integer repeatSeconds) + { + this.repeatSeconds = repeatSeconds; + } + + + + /******************************************************************************* + ** Fluent setter for repeatSeconds + ** + *******************************************************************************/ + public QScheduleMetaData withRepeatSeconds(Integer repeatSeconds) + { + this.repeatSeconds = repeatSeconds; + return (this); + } + + + + /******************************************************************************* + ** Getter for initialDelaySeconds + ** + *******************************************************************************/ + public Integer getInitialDelaySeconds() + { + return initialDelaySeconds; + } + + + + /******************************************************************************* + ** Setter for initialDelaySeconds + ** + *******************************************************************************/ + public void setInitialDelaySeconds(Integer initialDelaySeconds) + { + this.initialDelaySeconds = initialDelaySeconds; + } + + + + /******************************************************************************* + ** Fluent setter for initialDelaySeconds + ** + *******************************************************************************/ + public QScheduleMetaData withInitialDelaySeconds(Integer initialDelaySeconds) + { + this.initialDelaySeconds = initialDelaySeconds; + return (this); + } + + + + /******************************************************************************* + ** Getter for repeatMillis + ** + *******************************************************************************/ + public Integer getRepeatMillis() + { + return repeatMillis; + } + + + + /******************************************************************************* + ** Setter for repeatMillis + ** + *******************************************************************************/ + public void setRepeatMillis(Integer repeatMillis) + { + this.repeatMillis = repeatMillis; + } + + + + /******************************************************************************* + ** Fluent setter for repeatMillis + ** + *******************************************************************************/ + public QScheduleMetaData withRepeatMillis(Integer repeatMillis) + { + this.repeatMillis = repeatMillis; + return (this); + } + + + + /******************************************************************************* + ** Getter for initialDelayMillis + ** + *******************************************************************************/ + public Integer getInitialDelayMillis() + { + return initialDelayMillis; + } + + + + /******************************************************************************* + ** Setter for initialDelayMillis + ** + *******************************************************************************/ + public void setInitialDelayMillis(Integer initialDelayMillis) + { + this.initialDelayMillis = initialDelayMillis; + } + + + + /******************************************************************************* + ** Fluent setter for initialDelayMillis + ** + *******************************************************************************/ + public QScheduleMetaData withInitialDelayMillis(Integer initialDelayMillis) + { + this.initialDelayMillis = initialDelayMillis; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java new file mode 100644 index 00000000..f17d0a29 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java @@ -0,0 +1,321 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationRunner; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** QQQ Service (Singleton) that starts up repeating, scheduled jobs within QQQ. + ** + ** These include: + ** - Automation providers (which require polling) + ** - Queue pollers + ** - Scheduled processes. + ** + ** All of these jobs run using a "system session" - as defined by the sessionSupplier. + *******************************************************************************/ +public class ScheduleManager +{ + private static final Logger LOG = LogManager.getLogger(ScheduleManager.class); + + private static ScheduleManager scheduleManager = null; + private final QInstance qInstance; + + protected Supplier sessionSupplier; + + ///////////////////////////////////////////////////////////////////////////////////// + // for jobs that don't define a delay index, auto-stagger them, using this counter // + ///////////////////////////////////////////////////////////////////////////////////// + private int delayIndex = 0; + + private List executors = new ArrayList<>(); + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private ScheduleManager(QInstance qInstance) + { + this.qInstance = qInstance; + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static ScheduleManager getInstance(QInstance qInstance) + { + if(scheduleManager == null) + { + scheduleManager = new ScheduleManager(qInstance); + } + return (scheduleManager); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void start() + { + String propertyName = "qqq.scheduleManager.enabled"; + String propertyValue = System.getProperty(propertyName, ""); + if(propertyValue.equals("false")) + { + LOG.warn("Not starting ScheduleManager (per system property [" + propertyName + "=" + propertyValue + "])."); + } + + for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values()) + { + startQueueProvider(queueProvider); + } + + for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values()) + { + startAutomationProvider(automationProvider); + } + + for(QProcessMetaData process : qInstance.getProcesses().values()) + { + if(process.getSchedule() != null) + { + startProcess(process); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void startAutomationProvider(QAutomationProviderMetaData automationProvider) + { + PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, automationProvider.getName(), sessionSupplier); + StandardScheduledExecutor executor = new StandardScheduledExecutor(pollingAutomationRunner); + + QScheduleMetaData schedule = Objects.requireNonNullElseGet(automationProvider.getSchedule(), this::getDefaultSchedule); + + executor.setName(automationProvider.getName()); + setScheduleInExecutor(schedule, executor); + executor.start(); + + executors.add(executor); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void startQueueProvider(QQueueProviderMetaData queueProvider) + { + switch(queueProvider.getType()) + { + case SQS: + startSqsProvider((SQSQueueProviderMetaData) queueProvider); + break; + default: + throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void startSqsProvider(SQSQueueProviderMetaData queueProvider) + { + QInstance scheduleManagerQueueInstance = qInstance; + Supplier scheduleManagerSessionSupplier = sessionSupplier; + + for(QQueueMetaData queue : qInstance.getQueues().values()) + { + if(queueProvider.getName().equals(queue.getProviderName())) + { + SQSQueuePoller sqsQueuePoller = new SQSQueuePoller(); + sqsQueuePoller.setQueueProviderMetaData(queueProvider); + sqsQueuePoller.setQueueMetaData(queue); + sqsQueuePoller.setQInstance(scheduleManagerQueueInstance); + sqsQueuePoller.setSessionSupplier(scheduleManagerSessionSupplier); + + StandardScheduledExecutor executor = new StandardScheduledExecutor(sqsQueuePoller); + + QScheduleMetaData schedule = Objects.requireNonNullElseGet(queue.getSchedule(), + () -> Objects.requireNonNullElseGet(queueProvider.getSchedule(), + this::getDefaultSchedule)); + + executor.setName(queue.getName()); + setScheduleInExecutor(schedule, executor); + executor.start(); + + executors.add(executor); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void startProcess(QProcessMetaData process) + { + Runnable runProcess = () -> + { + try + { + RunProcessInput runProcessInput = new RunProcessInput(qInstance); + runProcessInput.setSession(sessionSupplier.get()); + runProcessInput.setProcessName(process.getName()); + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + + RunProcessAction runProcessAction = new RunProcessAction(); + runProcessAction.execute(runProcessInput); + } + catch(Exception e) + { + LOG.warn("Exception thrown running scheduled process [" + process.getName() + "]", e); + } + }; + + StandardScheduledExecutor executor = new StandardScheduledExecutor(runProcess); + executor.setName("process:" + process.getName()); + setScheduleInExecutor(process.getSchedule(), executor); + executor.start(); + + executors.add(executor); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setScheduleInExecutor(QScheduleMetaData schedule, StandardScheduledExecutor executor) + { + if(schedule.getRepeatMillis() != null) + { + executor.setDelayMillis(schedule.getRepeatMillis()); + } + else + { + executor.setDelayMillis(1000 * schedule.getRepeatSeconds()); + } + + if(schedule.getInitialDelayMillis() != null) + { + executor.setInitialDelayMillis(schedule.getInitialDelayMillis()); + } + else if(schedule.getInitialDelaySeconds() != null) + { + executor.setInitialDelayMillis(1000 * schedule.getInitialDelaySeconds()); + } + else + { + executor.setInitialDelayMillis(1000 * ++delayIndex); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QScheduleMetaData getDefaultSchedule() + { + QScheduleMetaData schedule; + schedule = new QScheduleMetaData() + .withInitialDelaySeconds(delayIndex++) + .withRepeatSeconds(60); + return schedule; + } + + + + /******************************************************************************* + ** Setter for sessionSupplier + ** + *******************************************************************************/ + public void setSessionSupplier(Supplier sessionSupplier) + { + this.sessionSupplier = sessionSupplier; + } + + + + /******************************************************************************* + ** Getter for managedExecutors + ** + *******************************************************************************/ + public List getExecutors() + { + return executors; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void stopAsync() + { + for(StandardScheduledExecutor scheduledExecutor : executors) + { + scheduledExecutor.stopAsync(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static void resetSingleton() + { + scheduleManager = null; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java similarity index 78% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java index 2837577c..7ae7d4dd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.actions.automation.polling; +package com.kingsrook.qqq.backend.core.scheduler; import java.util.concurrent.Executors; @@ -33,20 +33,20 @@ import org.apache.logging.log4j.Logger; /******************************************************************************* - ** Singleton that runs a Polling Automation Provider. Call its 'start' method - ** to make it go. Likely you need to set a sessionSupplier before you start - - ** so that threads that do work will have a valid session. + ** Standard class ran by ScheduleManager. Takes a Runnable in its constructor - + ** that's the code that actually executes. + ** *******************************************************************************/ -public class PollingAutomationExecutor +public class StandardScheduledExecutor { - private static final Logger LOG = LogManager.getLogger(PollingAutomationExecutor.class); - - private static PollingAutomationExecutor pollingAutomationExecutor = null; + private static final Logger LOG = LogManager.getLogger(StandardScheduledExecutor.class); private Integer initialDelayMillis = 3000; private Integer delayMillis = 1000; - private Supplier sessionSupplier; + protected QInstance qInstance; + protected String name; + protected Supplier sessionSupplier; private RunningState runningState = RunningState.STOPPED; private ScheduledExecutorService service; @@ -54,25 +54,29 @@ public class PollingAutomationExecutor /******************************************************************************* - ** Singleton constructor + ** Constructor + ** *******************************************************************************/ - private PollingAutomationExecutor() + public StandardScheduledExecutor(Runnable runnable) { - + this.runnable = runnable; } /******************************************************************************* - ** Singleton accessor + ** *******************************************************************************/ - public static PollingAutomationExecutor getInstance() + private Runnable runnable; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Runnable getRunnable() { - if(pollingAutomationExecutor == null) - { - pollingAutomationExecutor = new PollingAutomationExecutor(); - } - return (pollingAutomationExecutor); + return (runnable); } @@ -81,7 +85,7 @@ public class PollingAutomationExecutor ** ** @return true iff the schedule was started *******************************************************************************/ - public boolean start(QInstance instance, String providerName) + public boolean start() { if(!runningState.equals(RunningState.STOPPED)) { @@ -89,9 +93,9 @@ public class PollingAutomationExecutor return (false); } - LOG.info("Starting PollingAutomationExecutor"); + LOG.info("Starting [" + name + "]"); service = Executors.newSingleThreadScheduledExecutor(); - service.scheduleWithFixedDelay(new PollingAutomationRunner(instance, providerName, sessionSupplier), initialDelayMillis, delayMillis, TimeUnit.MILLISECONDS); + service.scheduleWithFixedDelay(getRunnable(), initialDelayMillis, delayMillis, TimeUnit.MILLISECONDS); runningState = RunningState.RUNNING; return (true); } @@ -121,7 +125,7 @@ public class PollingAutomationExecutor return (false); } - LOG.info("Stopping PollingAutomationExecutor"); + LOG.info("Stopping [" + name + "]"); runningState = RunningState.STOPPING; service.shutdown(); @@ -129,7 +133,7 @@ public class PollingAutomationExecutor { if(service.awaitTermination(300, TimeUnit.SECONDS)) { - LOG.info("Successfully stopped PollingAutomationExecutor"); + LOG.info("Successfully stopped [" + name + "]"); runningState = RunningState.STOPPED; return (true); } @@ -192,6 +196,28 @@ public class PollingAutomationExecutor + /******************************************************************************* + ** Setter for qInstance + ** + *******************************************************************************/ + public void setQInstance(QInstance qInstance) + { + this.qInstance = qInstance; + } + + + + /******************************************************************************* + ** Setter for name + ** + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + /******************************************************************************* ** Setter for sessionSupplier ** diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java similarity index 89% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java index faa3b0b1..37b2cce8 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; @@ -39,6 +40,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.scheduler.StandardScheduledExecutor; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; @@ -49,9 +51,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* - ** Unit test for PollingAutomationExecutor + ** Unit test for StandardScheduledExecutor *******************************************************************************/ -class PollingAutomationExecutorTest +class StandardScheduledExecutorTest { /******************************************************************************* @@ -89,7 +91,7 @@ class PollingAutomationExecutorTest //////////////////////////////////////////////// // have the polling executor run "for awhile" // //////////////////////////////////////////////// - runPollingAutomationExecutorForAwhile(qInstance); + runPollingAutomationExecutorForAwhile(qInstance, QSession::new); ///////////////////////////////////////////////// // query for the records - assert their status // @@ -112,8 +114,6 @@ class PollingAutomationExecutorTest - - /******************************************************************************* ** *******************************************************************************/ @@ -140,15 +140,14 @@ class PollingAutomationExecutorTest )); new InsertAction().execute(insertInput); - String uuid = UUID.randomUUID().toString(); + String uuid = UUID.randomUUID().toString(); QSession session = new QSession(); session.setIdReference(uuid); - PollingAutomationExecutor.getInstance().setSessionSupplier(() -> session); //////////////////////////////////////////////// // have the polling executor run "for awhile" // //////////////////////////////////////////////// - runPollingAutomationExecutorForAwhile(qInstance); + runPollingAutomationExecutorForAwhile(qInstance, () -> session); ///////////////////////////////////////////////////////////////////////////////////////////////////// // assert that the uuid we put in our session was present in the CaptureSessionIdAutomationHandler // @@ -183,12 +182,17 @@ class PollingAutomationExecutorTest /******************************************************************************* ** *******************************************************************************/ - private void runPollingAutomationExecutorForAwhile(QInstance qInstance) + private void runPollingAutomationExecutorForAwhile(QInstance qInstance, Supplier sessionSupplier) { - PollingAutomationExecutor pollingAutomationExecutor = PollingAutomationExecutor.getInstance(); + PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, TestUtils.POLLING_AUTOMATION, sessionSupplier); + + StandardScheduledExecutor pollingAutomationExecutor = new StandardScheduledExecutor(pollingAutomationRunner); pollingAutomationExecutor.setInitialDelayMillis(0); pollingAutomationExecutor.setDelayMillis(100); - pollingAutomationExecutor.start(qInstance, TestUtils.POLLING_AUTOMATION); + pollingAutomationExecutor.setQInstance(qInstance); + pollingAutomationExecutor.setName(TestUtils.POLLING_AUTOMATION); + pollingAutomationExecutor.setSessionSupplier(sessionSupplier); + pollingAutomationExecutor.start(); SleepUtils.sleep(1, TimeUnit.SECONDS); pollingAutomationExecutor.stop(); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java new file mode 100644 index 00000000..0a0de19f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java @@ -0,0 +1,131 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.util.List; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for ScheduleManager + *******************************************************************************/ +class ScheduleManagerTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + ScheduleManager.resetSingleton(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStartAndStop() + { + QInstance qInstance = TestUtils.defineInstance(); + ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance); + scheduleManager.start(); + + assertThat(scheduleManager.getExecutors()).isNotEmpty(); + + scheduleManager.stopAsync(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testScheduledProcess() throws QInstanceValidationException + { + QInstance qInstance = TestUtils.defineInstance(); + new QInstanceValidator().validate(qInstance); + qInstance.getAutomationProviders().clear(); + qInstance.getQueueProviders().clear(); + + qInstance.addProcess( + new QProcessMetaData() + .withName("testScheduledProcess") + .withSchedule(new QScheduleMetaData().withRepeatMillis(2).withInitialDelaySeconds(0)) + .withStepList(List.of(new QBackendStepMetaData() + .withName("step") + .withCode(new QCodeReference(BasicStep.class)))) + ); + + BasicStep.counter = 0; + + ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance); + scheduleManager.setSessionSupplier(QSession::new); + scheduleManager.start(); + SleepUtils.sleep(50, TimeUnit.MILLISECONDS); + scheduleManager.stopAsync(); + + System.out.println("Ran: " + BasicStep.counter + " times"); + assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class BasicStep implements BackendStep + { + static int counter = 0; + + + + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + counter++; + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index dcd96a15..d9cbd9d2 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; @@ -72,6 +73,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType; @@ -121,7 +125,8 @@ public class TestUtils public static final String POSSIBLE_VALUE_SOURCE_CUSTOM = "custom"; // custom-type public static final String POSSIBLE_VALUE_SOURCE_AUTOMATION_STATUS = "automationStatus"; - public static final String POLLING_AUTOMATION = "polling"; + public static final String POLLING_AUTOMATION = "polling"; + public static final String DEFAULT_QUEUE_PROVIDER = "defaultQueueProvider"; @@ -156,6 +161,9 @@ public class TestUtils qInstance.addAutomationProvider(definePollingAutomationProvider()); + qInstance.addQueueProvider(defineSqsProvider()); + qInstance.addQueue(defineTestSqsQueue()); + defineWidgets(qInstance); defineApps(qInstance); @@ -841,4 +849,41 @@ public class TestUtils return (new QPossibleValue<>(idValue, "Custom[" + idValue + "]")); } } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QQueueProviderMetaData defineSqsProvider() + { + QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter(); + + String accessKey = interpreter.interpret("${env.SQS_ACCESS_KEY}"); + String secretKey = interpreter.interpret("${env.SQS_SECRET_KEY}"); + String region = interpreter.interpret("${env.SQS_REGION}"); + String baseURL = interpreter.interpret("${env.SQS_BASE_URL}"); + + return (new SQSQueueProviderMetaData() + .withName(DEFAULT_QUEUE_PROVIDER) + .withAccessKey(accessKey) + .withSecretKey(secretKey) + .withRegion(region) + .withBaseURL(baseURL)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QQueueMetaData defineTestSqsQueue() + { + return (new QQueueMetaData() + .withName("testSQSQueue") + .withProviderName(DEFAULT_QUEUE_PROVIDER) + .withQueueName("test-queue") + .withProcessName("receiveEasypostTrackerWebhook")); + } + } diff --git a/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/EasyPostApiTest.java b/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/EasyPostApiTest.java index 48668dec..20cb55e6 100644 --- a/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/EasyPostApiTest.java +++ b/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/EasyPostApiTest.java @@ -45,6 +45,15 @@ public class EasyPostApiTest { /******************************************************************************* + ** Supported Tracking Numbers (and their statuses) + ** + ** EZ1000000001 : pre-transit + ** EZ2000000002 : in transit + ** EZ3000000003 : out for delivery + ** EZ4000000004 : delivered + ** EZ5000000005 : return to sender + ** EZ6000000006 : failure + ** EZ7000000007 : unknown ** *******************************************************************************/ @Test @@ -53,7 +62,7 @@ public class EasyPostApiTest QRecord record = new QRecord() .withValue("__ignoreMe", "123") .withValue("carrierCode", "USPS") - .withValue("trackingNo", "EZ1000000001"); + .withValue("trackingNo", "EZ4000000004"); InsertInput insertInput = new InsertInput(TestUtils.defineInstance()); insertInput.setSession(new QSession()); diff --git a/qqq-backend-module-rdbms/pom.xml b/qqq-backend-module-rdbms/pom.xml index 8617f4b0..4cee90bb 100644 --- a/qqq-backend-module-rdbms/pom.xml +++ b/qqq-backend-module-rdbms/pom.xml @@ -102,7 +102,7 @@ - package + ${plugin.shade.phase} shade diff --git a/qqq-middleware-lambda/pom.xml b/qqq-middleware-lambda/pom.xml index fda13437..1105dbf3 100644 --- a/qqq-middleware-lambda/pom.xml +++ b/qqq-middleware-lambda/pom.xml @@ -124,7 +124,7 @@ - package + ${plugin.shade.phase} shade diff --git a/qqq-utility-lambdas/README.md b/qqq-utility-lambdas/README.md new file mode 100644 index 00000000..b0ff6922 --- /dev/null +++ b/qqq-utility-lambdas/README.md @@ -0,0 +1,23 @@ +# qqq-middleware-javalin +This is a utility module, for use with QQQ applications, that want to +have simple, related AWS lambda functions - which aren't actually running +all of QQQ - but are helpful for integrating qqq with lambdas. + +## License +QQQ - Low-code Application Framework for Engineers. \ +Copyright (C) 2022. Kingsrook, LLC \ +651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States \ +contact@kingsrook.com | https://github.com/Kingsrook/ + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . diff --git a/qqq-utility-lambdas/pom.xml b/qqq-utility-lambdas/pom.xml new file mode 100644 index 00000000..511f37a4 --- /dev/null +++ b/qqq-utility-lambdas/pom.xml @@ -0,0 +1,286 @@ + + + + + 4.0.0 + + qqq-utility-lambdas + + + com.kingsrook.qqq + qqq-parent-project + ${revision} + + + + + 11 + 11 + + + + + + com.amazonaws + aws-lambda-java-core + 1.2.1 + + + com.amazonaws + aws-java-sdk-sqs + 1.12.321 + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + -Xlint:unchecked + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + + @{jaCoCoArgLine} + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.1.0 + + true + resolveCiFriendliesOnly + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + com.amashchenko.maven.plugin + gitflow-maven-plugin + 1.18.0 + + + main + dev + version- + + true + install + true + 1 + revision + true + + + + org.jacoco + jacoco-maven-plugin + 0.8.8 + + + pre-unit-test + + prepare-agent + + + jaCoCoArgLine + + + + unit-test-check + + check + + + + ${coverage.haltOnFailure} + + + BUNDLE + + + INSTRUCTION + COVEREDRATIO + ${coverage.instructionCoveredRatioMinimum} + + + CLASS + COVEREDRATIO + ${coverage.classCoveredRatioMinimum} + + + + + + + + post-unit-test + verify + + report + + + + + + exec-maven-plugin + org.codehaus.mojo + 3.0.0 + + + test-coverage-summary + verify + + exec + + + sh + + -c + + /dev/null 2>&1 +if [ "$?" == "0" ]; then + echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers + xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values + paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}' + rm /tmp/$$.headers /tmp/$$.values +else + echo "xpath is not installed. Jacoco coverage summary will not be produced here.."; +fi + ]]> + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + false + + + *:* + + META-INF/* + + + + + + + package + + shade + + + + + + + + + + github-qqq-maven-registry + GitHub QQQ Maven Registry + https://maven.pkg.github.com/Kingsrook/qqq-maven-registry + + + + + + github-qqq-maven-registry + GitHub QQQ Maven Registry + https://maven.pkg.github.com/Kingsrook/qqq-maven-registry + + + + diff --git a/qqq-utility-lambdas/src/main/java/com/kingsrook/qqq/utilitylambdas/QPostToSQSLambda.java b/qqq-utility-lambdas/src/main/java/com/kingsrook/qqq/utilitylambdas/QPostToSQSLambda.java new file mode 100644 index 00000000..b5a0e627 --- /dev/null +++ b/qqq-utility-lambdas/src/main/java/com/kingsrook/qqq/utilitylambdas/QPostToSQSLambda.java @@ -0,0 +1,128 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.utilitylambdas; + + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestStreamHandler; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.services.sqs.model.SendMessageRequest; + + +/******************************************************************************* + ** QQQ Utility Lambda to post input data to SQS. + ** + ** Requires environment variable: QUEUE_URL (e.g., https://sqs.us-east-0.amazonaws.com/111122223333/my-queue-name) + *******************************************************************************/ +public class QPostToSQSLambda implements RequestStreamHandler +{ + protected Context context; + + + + /******************************************************************************* + ** Entrypoint from AWS Lambda. + *******************************************************************************/ + public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException + { + this.context = context; + + try + { + String input = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); + log("Full Input: " + input); + + final AmazonSQS sqs = AmazonSQSClientBuilder.defaultClient(); + + SendMessageRequest sendMessageRequest = new SendMessageRequest() + .withQueueUrl(System.getenv("QUEUE_URL")) + .withMessageBody(input); + sqs.sendMessage(sendMessageRequest); + + writeResponse(outputStream, "OK"); + } + catch(Exception e) + { + log(e); + throw (new IOException(e)); + // writeResponse(outputStream, requestId, 500, "Uncaught error handing request: " + e.getMessage()); + } + } + + + + /******************************************************************************* + ** Write to the cloudwatch logs. + *******************************************************************************/ + protected void log(String message) + { + if(message == null) + { + return; + } + + context.getLogger().log(message + (message.endsWith("\n") ? "" : "\n")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void log(Throwable t) + { + if(t == null) + { + return; + } + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + t.printStackTrace(pw); + log(sw + "\n"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void writeResponse(OutputStream outputStream, String messageBody) throws IOException + { + StringBuilder body = new StringBuilder("{"); + if(messageBody != null && !messageBody.equals("")) + { + body.append("\"body\":\"").append(messageBody.replaceAll("\"", "'")).append("\""); + } + body.append("}"); + + outputStream.write(body.toString().getBytes(StandardCharsets.UTF_8)); + } + +}