QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-17 04:30:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding queues and queue providers; Adding schedules and ScheduleManager

Browse Source
This commit is contained in:
darin.kelkhoff
2022-10-18 09:00:21 -05:00
parent 888239b265
commit 6345846eba
23 changed files with 2172 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
pom.xml
View File

@ -36,6 +36,7 @@
<module>qqq-middleware-picocli</module>
<module>qqq-middleware-javalin</module>
<module>qqq-middleware-lambda</module>
<module>qqq-utility-lambdas</module>
<module>qqq-sample-project</module>
</modules>
@ -51,8 +52,20 @@
<coverage.haltOnFailure>true</coverage.haltOnFailure>
<coverage.instructionCoveredRatioMinimum>0.80</coverage.instructionCoveredRatioMinimum>
<coverage.classCoveredRatioMinimum>0.95</coverage.classCoveredRatioMinimum>
<plugin.shade.phase>none</plugin.shade.phase>
</properties>
<profiles>
<profile>
<!-- For qqq-middleware-lambda - to build its shaded jar, its qqq dependencies also need
to build a shaded jar. So to activate that mode, use this profile (-P buildShadedJar)-->
<id>buildShadedJar</id>
<properties>
<plugin.shade.phase>package</plugin.shade.phase>
</properties>
</profile>
</profiles>
<dependencyManagement>
<dependencies>
<dependency>
@ -88,7 +101,7 @@
<version>3.23.1</version>
<scope>test</scope>
</dependency>
</dependencies>
</dependencies>
</dependencyManagement>
<build>

8
qqq-backend-core/pom.xml
View File

@ -115,6 +115,12 @@
<version>v3-rev20220815-2.0.0</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-sqs</artifactId>
<version>1.12.321</version>
</dependency>
<!-- Common deps for all qqq modules -->
<dependency>
<groupId>org.apache.maven.plugins</groupId>
@ -175,7 +181,7 @@
</configuration>
<executions>
<execution>
<phase>package</phase>
<phase>${plugin.shade.phase}</phase>
<goals>
<goal>shade</goal>
</goals>

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java
View File

@ -67,7 +67,7 @@ import org.apache.logging.log4j.Logger;
** Runnable for the Polling Automation Provider, that looks for records that
** need automations, and executes them.
*******************************************************************************/
class PollingAutomationRunner implements Runnable
public class PollingAutomationRunner implements Runnable
{
private static final Logger LOG = LogManager.getLogger(PollingAutomationRunner.class);

167
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/queues/SQSQueuePoller.java Normal file
View File

@ -0,0 +1,167 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.queues;
import java.util.function.Supplier;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
import com.amazonaws.services.sqs.model.DeleteMessageRequest;
import com.amazonaws.services.sqs.model.Message;
import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
import com.amazonaws.services.sqs.model.ReceiveMessageResult;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Class to poll an SQS queue, and run process code for each message found.
*******************************************************************************/
public class SQSQueuePoller implements Runnable
{
private static final Logger LOG = LogManager.getLogger(SQSQueuePoller.class);
///////////////////////////////////////////////
// todo - move these 2 to a "QBaseRunnable"? //
///////////////////////////////////////////////
private QInstance qInstance;
private Supplier<QSession> sessionSupplier;
private SQSQueueProviderMetaData queueProviderMetaData;
private QQueueMetaData queueMetaData;
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run()
{
try
{
BasicAWSCredentials credentials = new BasicAWSCredentials(queueProviderMetaData.getAccessKey(), queueProviderMetaData.getSecretKey());
final AmazonSQS sqs = AmazonSQSClientBuilder.standard()
.withRegion(queueProviderMetaData.getRegion())
.withCredentials(new AWSStaticCredentialsProvider(credentials))
.build();
String queueUrl = queueProviderMetaData.getBaseURL();
if(!queueUrl.endsWith("/"))
{
queueUrl += "/";
}
queueUrl += queueMetaData.getQueueName();
while(true)
{
ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest();
receiveMessageRequest.setQueueUrl(queueUrl);
ReceiveMessageResult receiveMessageResult = sqs.receiveMessage(receiveMessageRequest);
if(receiveMessageResult.getMessages().isEmpty())
{
LOG.debug("0 messages received. Breaking.");
break;
}
LOG.debug(receiveMessageResult.getMessages().size() + " messages received. Processing.");
for(Message message : receiveMessageResult.getMessages())
{
String body = message.getBody();
RunProcessInput runProcessInput = new RunProcessInput(qInstance);
runProcessInput.setSession(sessionSupplier.get());
runProcessInput.setProcessName(queueMetaData.getProcessName());
runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
runProcessInput.addValue("body", body);
RunProcessAction runProcessAction = new RunProcessAction();
RunProcessOutput runProcessOutput = runProcessAction.execute(runProcessInput);
/////////////////////////////////
// todo - what of exceptions?? //
/////////////////////////////////
String receiptHandle = message.getReceiptHandle();
sqs.deleteMessage(new DeleteMessageRequest(queueUrl, receiptHandle));
}
}
}
catch(Exception e)
{
LOG.warn("Error receiving SQS Message", e);
}
}
/*******************************************************************************
** Setter for queueProviderMetaData
**
*******************************************************************************/
public void setQueueProviderMetaData(SQSQueueProviderMetaData queueProviderMetaData)
{
this.queueProviderMetaData = queueProviderMetaData;
}
/*******************************************************************************
** Setter for queueMetaData
**
*******************************************************************************/
public void setQueueMetaData(QQueueMetaData queueMetaData)
{
this.queueMetaData = queueMetaData;
}
/*******************************************************************************
** Setter for qInstance
**
*******************************************************************************/
public void setQInstance(QInstance qInstance)
{
this.qInstance = qInstance;
}
/*******************************************************************************
** Setter for sessionSupplier
**
*******************************************************************************/
public void setSessionSupplier(Supplier<QSession> sessionSupplier)
{
this.sessionSupplier = sessionSupplier;
}
}

125
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java
View File

@ -36,6 +36,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
@ -69,6 +71,9 @@ public class QInstance
private Map<String, QWidgetMetaDataInterface> widgets = new LinkedHashMap<>();
private Map<String, QQueueProviderMetaData> queueProviders = new LinkedHashMap<>();
private Map<String, QQueueMetaData> queues = new LinkedHashMap<>();
// todo - lock down the object (no more changes allowed) after it's been validated?
@JsonIgnore
@ -670,4 +675,124 @@ public class QInstance
return (this.widgets.get(name));
}
/*******************************************************************************
**
*******************************************************************************/
public void addQueueProvider(QQueueProviderMetaData queueProvider)
{
this.addQueueProvider(queueProvider.getName(), queueProvider);
}
/*******************************************************************************
**
*******************************************************************************/
public void addQueueProvider(String name, QQueueProviderMetaData queueProvider)
{
if(!StringUtils.hasContent(name))
{
throw (new IllegalArgumentException("Attempted to add an queueProvider without a name."));
}
if(this.queueProviders.containsKey(name))
{
throw (new IllegalArgumentException("Attempted to add a second queueProvider with name: " + name));
}
this.queueProviders.put(name, queueProvider);
}
/*******************************************************************************
**
*******************************************************************************/
public QQueueProviderMetaData getQueueProvider(String name)
{
return (this.queueProviders.get(name));
}
/*******************************************************************************
** Getter for queueProviders
**
*******************************************************************************/
public Map<String, QQueueProviderMetaData> getQueueProviders()
{
return queueProviders;
}
/*******************************************************************************
** Setter for queueProviders
**
*******************************************************************************/
public void setQueueProviders(Map<String, QQueueProviderMetaData> queueProviders)
{
this.queueProviders = queueProviders;
}
/*******************************************************************************
**
*******************************************************************************/
public void addQueue(QQueueMetaData queue)
{
this.addQueue(queue.getName(), queue);
}
/*******************************************************************************
**
*******************************************************************************/
public void addQueue(String name, QQueueMetaData queue)
{
if(!StringUtils.hasContent(name))
{
throw (new IllegalArgumentException("Attempted to add an queue without a name."));
}
if(this.queues.containsKey(name))
{
throw (new IllegalArgumentException("Attempted to add a second queue with name: " + name));
}
this.queues.put(name, queue);
}
/*******************************************************************************
**
*******************************************************************************/
public QQueueMetaData getQueue(String name)
{
return (this.queues.get(name));
}
/*******************************************************************************
** Getter for queues
**
*******************************************************************************/
public Map<String, QQueueMetaData> getQueues()
{
return queues;
}
/*******************************************************************************
** Setter for queues
**
*******************************************************************************/
public void setQueues(Map<String, QQueueMetaData> queues)
{
this.queues = queues;
}
}

44
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java
View File

@ -22,14 +22,20 @@
package com.kingsrook.qqq.backend.core.model.metadata.automation;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
/*******************************************************************************
** Meta-data definition of a qqq service to drive record automations.
*******************************************************************************/
public class QAutomationProviderMetaData
{
private String name;
private String name;
private QAutomationProviderType type;
private QScheduleMetaData schedule;
/*******************************************************************************
** Getter for name
@ -52,6 +58,7 @@ public class QAutomationProviderMetaData
}
/*******************************************************************************
** Fluent setter for name
**
@ -85,6 +92,7 @@ public class QAutomationProviderMetaData
}
/*******************************************************************************
** Fluent setter for type
**
@ -95,4 +103,38 @@ public class QAutomationProviderMetaData
return (this);
}
/*******************************************************************************
** Getter for schedule
**
*******************************************************************************/
public QScheduleMetaData getSchedule()
{
return schedule;
}
/*******************************************************************************
** Setter for schedule
**
*******************************************************************************/
public void setSchedule(QScheduleMetaData schedule)
{
this.schedule = schedule;
}
/*******************************************************************************
** Fluent setter for schedule
**
*******************************************************************************/
public QAutomationProviderMetaData withSchedule(QScheduleMetaData schedule)
{
this.schedule = schedule;
return (this);
}
}

37
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java
View File

@ -32,6 +32,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
/*******************************************************************************
@ -51,6 +52,8 @@ public class QProcessMetaData implements QAppChildMetaData
private String parentAppName;
private QIcon icon;
private QScheduleMetaData schedule;
/*******************************************************************************
@ -436,4 +439,38 @@ public class QProcessMetaData implements QAppChildMetaData
return (this);
}
/*******************************************************************************
** Getter for schedule
**
*******************************************************************************/
public QScheduleMetaData getSchedule()
{
return schedule;
}
/*******************************************************************************
** Setter for schedule
**
*******************************************************************************/
public void setSchedule(QScheduleMetaData schedule)
{
this.schedule = schedule;
}
/*******************************************************************************
** Fluent setter for schedule
**
*******************************************************************************/
public QProcessMetaData withSchedule(QScheduleMetaData schedule)
{
this.schedule = schedule;
return (this);
}
}

216
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QQueueMetaData.java Normal file
View File

@ -0,0 +1,216 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.queues;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
/*******************************************************************************
** MetaData to define a message queue, which must exist within a QueueProvider.
**
** The name attribute is a globally unique name within the QInstance
** The providerName is the connection to the queue system.
** The queueName uniquely identifies the queue within the context of the provider.
** The processName is the code that runs for messages found on the queue.
** The schedule may not be used by all provider types, but defines when the queue is polled.
*******************************************************************************/
public class QQueueMetaData
{
private String name;
private String providerName;
private String queueName;
private String processName;
private QScheduleMetaData schedule;
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Setter for name
**
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public QQueueMetaData withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Getter for providerName
**
*******************************************************************************/
public String getProviderName()
{
return providerName;
}
/*******************************************************************************
** Setter for providerName
**
*******************************************************************************/
public void setProviderName(String providerName)
{
this.providerName = providerName;
}
/*******************************************************************************
** Fluent setter for providerName
**
*******************************************************************************/
public QQueueMetaData withProviderName(String providerName)
{
this.providerName = providerName;
return (this);
}
/*******************************************************************************
** Getter for queueName
**
*******************************************************************************/
public String getQueueName()
{
return queueName;
}
/*******************************************************************************
** Setter for queueName
**
*******************************************************************************/
public void setQueueName(String queueName)
{
this.queueName = queueName;
}
/*******************************************************************************
** Fluent setter for queueName
**
*******************************************************************************/
public QQueueMetaData withQueueName(String queueName)
{
this.queueName = queueName;
return (this);
}
/*******************************************************************************
** Getter for processName
**
*******************************************************************************/
public String getProcessName()
{
return processName;
}
/*******************************************************************************
** Setter for processName
**
*******************************************************************************/
public void setProcessName(String processName)
{
this.processName = processName;
}
/*******************************************************************************
** Fluent setter for processName
**
*******************************************************************************/
public QQueueMetaData withProcessName(String processName)
{
this.processName = processName;
return (this);
}
/*******************************************************************************
** Getter for schedule
**
*******************************************************************************/
public QScheduleMetaData getSchedule()
{
return schedule;
}
/*******************************************************************************
** Setter for schedule
**
*******************************************************************************/
public void setSchedule(QScheduleMetaData schedule)
{
this.schedule = schedule;
}
/*******************************************************************************
** Fluent setter for schedule
**
*******************************************************************************/
public QQueueMetaData withSchedule(QScheduleMetaData schedule)
{
this.schedule = schedule;
return (this);
}
}

101
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QQueueProviderMetaData.java Normal file
View File

@ -0,0 +1,101 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.queues;
/*******************************************************************************
** Define a provider of queues (e.g., an MQ system, or SQS)
*******************************************************************************/
public class QQueueProviderMetaData
{
private String name;
private QueueType type;
/*******************************************************************************
** Getter for name
**
*******************************************************************************/
public String getName()
{
return name;
}
/*******************************************************************************
** Setter for name
**
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Fluent setter for name
**
*******************************************************************************/
public QQueueProviderMetaData withName(String name)
{
this.name = name;
return (this);
}
/*******************************************************************************
** Getter for type
**
*******************************************************************************/
public QueueType getType()
{
return type;
}
/*******************************************************************************
** Setter for type
**
*******************************************************************************/
public void setType(QueueType type)
{
this.type = type;
}
/*******************************************************************************
** Fluent setter for type
**
*******************************************************************************/
public QQueueProviderMetaData withType(QueueType type)
{
this.type = type;
return (this);
}
}

34
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QueueType.java Normal file
View File

@ -0,0 +1,34 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.queues;
/*******************************************************************************
** Types of queues supported by QQQ.
*******************************************************************************/
public enum QueueType
{
SQS
// todo MQ
// todo? IN_MEMORY
// todo? TABLE
}

238
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java Normal file
View File

@ -0,0 +1,238 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.queues;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
/*******************************************************************************
** Meta-data for an source of Amazon SQS queues (e.g, an aws account/credential
** set, with a common base URL).
**
** Scheduled can be defined here, to apply to all queues in the provider - or
** each can supply their own schedule.
*******************************************************************************/
public class SQSQueueProviderMetaData extends QQueueProviderMetaData
{
private String accessKey;
private String secretKey;
private String region;
private String baseURL;
private QScheduleMetaData schedule;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public SQSQueueProviderMetaData()
{
super();
setType(QueueType.SQS);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public SQSQueueProviderMetaData withName(String name)
{
super.withName(name);
return (this);
}
/*******************************************************************************
** Getter for accessKey
**
*******************************************************************************/
public String getAccessKey()
{
return accessKey;
}
/*******************************************************************************
** Setter for accessKey
**
*******************************************************************************/
public void setAccessKey(String accessKey)
{
this.accessKey = accessKey;
}
/*******************************************************************************
** Fluent setter for accessKey
**
*******************************************************************************/
public SQSQueueProviderMetaData withAccessKey(String accessKey)
{
this.accessKey = accessKey;
return (this);
}
/*******************************************************************************
** Getter for secretKey
**
*******************************************************************************/
public String getSecretKey()
{
return secretKey;
}
/*******************************************************************************
** Setter for secretKey
**
*******************************************************************************/
public void setSecretKey(String secretKey)
{
this.secretKey = secretKey;
}
/*******************************************************************************
** Fluent setter for secretKey
**
*******************************************************************************/
public SQSQueueProviderMetaData withSecretKey(String secretKey)
{
this.secretKey = secretKey;
return (this);
}
/*******************************************************************************
** Getter for region
**
*******************************************************************************/
public String getRegion()
{
return region;
}
/*******************************************************************************
** Setter for region
**
*******************************************************************************/
public void setRegion(String region)
{
this.region = region;
}
/*******************************************************************************
** Fluent setter for region
**
*******************************************************************************/
public SQSQueueProviderMetaData withRegion(String region)
{
this.region = region;
return (this);
}
/*******************************************************************************
** Getter for baseURL
**
*******************************************************************************/
public String getBaseURL()
{
return baseURL;
}
/*******************************************************************************
** Setter for baseURL
**
*******************************************************************************/
public void setBaseURL(String baseURL)
{
this.baseURL = baseURL;
}
/*******************************************************************************
** Fluent setter for baseURL
**
*******************************************************************************/
public SQSQueueProviderMetaData withBaseURL(String baseURL)
{
this.baseURL = baseURL;
return (this);
}
/*******************************************************************************
** Getter for schedule
**
*******************************************************************************/
public QScheduleMetaData getSchedule()
{
return schedule;
}
/*******************************************************************************
** Setter for schedule
**
*******************************************************************************/
public void setSchedule(QScheduleMetaData schedule)
{
this.schedule = schedule;
}
/*******************************************************************************
** Fluent setter for schedule
**
*******************************************************************************/
public SQSQueueProviderMetaData withSchedule(QScheduleMetaData schedule)
{
this.schedule = schedule;
return (this);
}
}

177
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java Normal file
View File

@ -0,0 +1,177 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.scheduleing;
/*******************************************************************************
** Meta-data to define scheduled actions within QQQ.
**
** Initially, only supports repeating jobs, either on a given # of seconds or millis.
** Can also specify an initialDelay - e.g., to avoid all jobs starting up at the
** same moment.
**
** In the future we most likely would want to allow cron strings to be added here.
*******************************************************************************/
public class QScheduleMetaData
{
private Integer repeatSeconds;
private Integer repeatMillis;
private Integer initialDelaySeconds;
private Integer initialDelayMillis;
/*******************************************************************************
** Getter for repeatSeconds
**
*******************************************************************************/
public Integer getRepeatSeconds()
{
return repeatSeconds;
}
/*******************************************************************************
** Setter for repeatSeconds
**
*******************************************************************************/
public void setRepeatSeconds(Integer repeatSeconds)
{
this.repeatSeconds = repeatSeconds;
}
/*******************************************************************************
** Fluent setter for repeatSeconds
**
*******************************************************************************/
public QScheduleMetaData withRepeatSeconds(Integer repeatSeconds)
{
this.repeatSeconds = repeatSeconds;
return (this);
}
/*******************************************************************************
** Getter for initialDelaySeconds
**
*******************************************************************************/
public Integer getInitialDelaySeconds()
{
return initialDelaySeconds;
}
/*******************************************************************************
** Setter for initialDelaySeconds
**
*******************************************************************************/
public void setInitialDelaySeconds(Integer initialDelaySeconds)
{
this.initialDelaySeconds = initialDelaySeconds;
}
/*******************************************************************************
** Fluent setter for initialDelaySeconds
**
*******************************************************************************/
public QScheduleMetaData withInitialDelaySeconds(Integer initialDelaySeconds)
{
this.initialDelaySeconds = initialDelaySeconds;
return (this);
}
/*******************************************************************************
** Getter for repeatMillis
**
*******************************************************************************/
public Integer getRepeatMillis()
{
return repeatMillis;
}
/*******************************************************************************
** Setter for repeatMillis
**
*******************************************************************************/
public void setRepeatMillis(Integer repeatMillis)
{
this.repeatMillis = repeatMillis;
}
/*******************************************************************************
** Fluent setter for repeatMillis
**
*******************************************************************************/
public QScheduleMetaData withRepeatMillis(Integer repeatMillis)
{
this.repeatMillis = repeatMillis;
return (this);
}
/*******************************************************************************
** Getter for initialDelayMillis
**
*******************************************************************************/
public Integer getInitialDelayMillis()
{
return initialDelayMillis;
}
/*******************************************************************************
** Setter for initialDelayMillis
**
*******************************************************************************/
public void setInitialDelayMillis(Integer initialDelayMillis)
{
this.initialDelayMillis = initialDelayMillis;
}
/*******************************************************************************
** Fluent setter for initialDelayMillis
**
*******************************************************************************/
public QScheduleMetaData withInitialDelayMillis(Integer initialDelayMillis)
{
this.initialDelayMillis = initialDelayMillis;
return (this);
}
}

321
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java Normal file
View File

@ -0,0 +1,321 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.scheduler;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationRunner;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.actions.queues.SQSQueuePoller;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** QQQ Service (Singleton) that starts up repeating, scheduled jobs within QQQ.
**
** These include:
** - Automation providers (which require polling)
** - Queue pollers
** - Scheduled processes.
**
** All of these jobs run using a "system session" - as defined by the sessionSupplier.
*******************************************************************************/
public class ScheduleManager
{
private static final Logger LOG = LogManager.getLogger(ScheduleManager.class);
private static ScheduleManager scheduleManager = null;
private final QInstance qInstance;
protected Supplier<QSession> sessionSupplier;
/////////////////////////////////////////////////////////////////////////////////////
// for jobs that don't define a delay index, auto-stagger them, using this counter //
/////////////////////////////////////////////////////////////////////////////////////
private int delayIndex = 0;
private List<StandardScheduledExecutor> executors = new ArrayList<>();
/*******************************************************************************
** Singleton constructor
*******************************************************************************/
private ScheduleManager(QInstance qInstance)
{
this.qInstance = qInstance;
}
/*******************************************************************************
** Singleton accessor
*******************************************************************************/
public static ScheduleManager getInstance(QInstance qInstance)
{
if(scheduleManager == null)
{
scheduleManager = new ScheduleManager(qInstance);
}
return (scheduleManager);
}
/*******************************************************************************
**
*******************************************************************************/
public void start()
{
String propertyName = "qqq.scheduleManager.enabled";
String propertyValue = System.getProperty(propertyName, "");
if(propertyValue.equals("false"))
{
LOG.warn("Not starting ScheduleManager (per system property [" + propertyName + "=" + propertyValue + "]).");
}
for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values())
{
startQueueProvider(queueProvider);
}
for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values())
{
startAutomationProvider(automationProvider);
}
for(QProcessMetaData process : qInstance.getProcesses().values())
{
if(process.getSchedule() != null)
{
startProcess(process);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private void startAutomationProvider(QAutomationProviderMetaData automationProvider)
{
PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, automationProvider.getName(), sessionSupplier);
StandardScheduledExecutor executor = new StandardScheduledExecutor(pollingAutomationRunner);
QScheduleMetaData schedule = Objects.requireNonNullElseGet(automationProvider.getSchedule(), this::getDefaultSchedule);
executor.setName(automationProvider.getName());
setScheduleInExecutor(schedule, executor);
executor.start();
executors.add(executor);
}
/*******************************************************************************
**
*******************************************************************************/
private void startQueueProvider(QQueueProviderMetaData queueProvider)
{
switch(queueProvider.getType())
{
case SQS:
startSqsProvider((SQSQueueProviderMetaData) queueProvider);
break;
default:
throw new IllegalArgumentException("Unhandled queue provider type: " + queueProvider.getType());
}
}
/*******************************************************************************
**
*******************************************************************************/
private void startSqsProvider(SQSQueueProviderMetaData queueProvider)
{
QInstance scheduleManagerQueueInstance = qInstance;
Supplier<QSession> scheduleManagerSessionSupplier = sessionSupplier;
for(QQueueMetaData queue : qInstance.getQueues().values())
{
if(queueProvider.getName().equals(queue.getProviderName()))
{
SQSQueuePoller sqsQueuePoller = new SQSQueuePoller();
sqsQueuePoller.setQueueProviderMetaData(queueProvider);
sqsQueuePoller.setQueueMetaData(queue);
sqsQueuePoller.setQInstance(scheduleManagerQueueInstance);
sqsQueuePoller.setSessionSupplier(scheduleManagerSessionSupplier);
StandardScheduledExecutor executor = new StandardScheduledExecutor(sqsQueuePoller);
QScheduleMetaData schedule = Objects.requireNonNullElseGet(queue.getSchedule(),
() -> Objects.requireNonNullElseGet(queueProvider.getSchedule(),
this::getDefaultSchedule));
executor.setName(queue.getName());
setScheduleInExecutor(schedule, executor);
executor.start();
executors.add(executor);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private void startProcess(QProcessMetaData process)
{
Runnable runProcess = () ->
{
try
{
RunProcessInput runProcessInput = new RunProcessInput(qInstance);
runProcessInput.setSession(sessionSupplier.get());
runProcessInput.setProcessName(process.getName());
runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
RunProcessAction runProcessAction = new RunProcessAction();
runProcessAction.execute(runProcessInput);
}
catch(Exception e)
{
LOG.warn("Exception thrown running scheduled process [" + process.getName() + "]", e);
}
};
StandardScheduledExecutor executor = new StandardScheduledExecutor(runProcess);
executor.setName("process:" + process.getName());
setScheduleInExecutor(process.getSchedule(), executor);
executor.start();
executors.add(executor);
}
/*******************************************************************************
**
*******************************************************************************/
private void setScheduleInExecutor(QScheduleMetaData schedule, StandardScheduledExecutor executor)
{
if(schedule.getRepeatMillis() != null)
{
executor.setDelayMillis(schedule.getRepeatMillis());
}
else
{
executor.setDelayMillis(1000 * schedule.getRepeatSeconds());
}
if(schedule.getInitialDelayMillis() != null)
{
executor.setInitialDelayMillis(schedule.getInitialDelayMillis());
}
else if(schedule.getInitialDelaySeconds() != null)
{
executor.setInitialDelayMillis(1000 * schedule.getInitialDelaySeconds());
}
else
{
executor.setInitialDelayMillis(1000 * ++delayIndex);
}
}
/*******************************************************************************
**
*******************************************************************************/
private QScheduleMetaData getDefaultSchedule()
{
QScheduleMetaData schedule;
schedule = new QScheduleMetaData()
.withInitialDelaySeconds(delayIndex++)
.withRepeatSeconds(60);
return schedule;
}
/*******************************************************************************
** Setter for sessionSupplier
**
*******************************************************************************/
public void setSessionSupplier(Supplier<QSession> sessionSupplier)
{
this.sessionSupplier = sessionSupplier;
}
/*******************************************************************************
** Getter for managedExecutors
**
*******************************************************************************/
public List<StandardScheduledExecutor> getExecutors()
{
return executors;
}
/*******************************************************************************
**
*******************************************************************************/
public void stopAsync()
{
for(StandardScheduledExecutor scheduledExecutor : executors)
{
scheduledExecutor.stopAsync();
}
}
/*******************************************************************************
**
*******************************************************************************/
static void resetSingleton()
{
scheduleManager = null;
}
}

74
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/StandardScheduledExecutor.java
View File

@ -19,7 +19,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.automation.polling;
package com.kingsrook.qqq.backend.core.scheduler;
import java.util.concurrent.Executors;
@ -33,20 +33,20 @@ import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Singleton that runs a Polling Automation Provider. Call its 'start' method
** to make it go. Likely you need to set a sessionSupplier before you start -
** so that threads that do work will have a valid session.
** Standard class ran by ScheduleManager. Takes a Runnable in its constructor -
** that's the code that actually executes.
**
*******************************************************************************/
public class PollingAutomationExecutor
public class StandardScheduledExecutor
{
private static final Logger LOG = LogManager.getLogger(PollingAutomationExecutor.class);
private static PollingAutomationExecutor pollingAutomationExecutor = null;
private static final Logger LOG = LogManager.getLogger(StandardScheduledExecutor.class);
private Integer initialDelayMillis = 3000;
private Integer delayMillis = 1000;
private Supplier<QSession> sessionSupplier;
protected QInstance qInstance;
protected String name;
protected Supplier<QSession> sessionSupplier;
private RunningState runningState = RunningState.STOPPED;
private ScheduledExecutorService service;
@ -54,25 +54,29 @@ public class PollingAutomationExecutor
/*******************************************************************************
** Singleton constructor
** Constructor
**
*******************************************************************************/
private PollingAutomationExecutor()
public StandardScheduledExecutor(Runnable runnable)
{
this.runnable = runnable;
}
/*******************************************************************************
** Singleton accessor
**
*******************************************************************************/
public static PollingAutomationExecutor getInstance()
private Runnable runnable;
/*******************************************************************************
**
*******************************************************************************/
public Runnable getRunnable()
{
if(pollingAutomationExecutor == null)
{
pollingAutomationExecutor = new PollingAutomationExecutor();
}
return (pollingAutomationExecutor);
return (runnable);
}
@ -81,7 +85,7 @@ public class PollingAutomationExecutor
**
** @return true iff the schedule was started
*******************************************************************************/
public boolean start(QInstance instance, String providerName)
public boolean start()
{
if(!runningState.equals(RunningState.STOPPED))
{
@ -89,9 +93,9 @@ public class PollingAutomationExecutor
return (false);
}
LOG.info("Starting PollingAutomationExecutor");
LOG.info("Starting [" + name + "]");
service = Executors.newSingleThreadScheduledExecutor();
service.scheduleWithFixedDelay(new PollingAutomationRunner(instance, providerName, sessionSupplier), initialDelayMillis, delayMillis, TimeUnit.MILLISECONDS);
service.scheduleWithFixedDelay(getRunnable(), initialDelayMillis, delayMillis, TimeUnit.MILLISECONDS);
runningState = RunningState.RUNNING;
return (true);
}
@ -121,7 +125,7 @@ public class PollingAutomationExecutor
return (false);
}
LOG.info("Stopping PollingAutomationExecutor");
LOG.info("Stopping [" + name + "]");
runningState = RunningState.STOPPING;
service.shutdown();
@ -129,7 +133,7 @@ public class PollingAutomationExecutor
{
if(service.awaitTermination(300, TimeUnit.SECONDS))
{
LOG.info("Successfully stopped PollingAutomationExecutor");
LOG.info("Successfully stopped [" + name + "]");
runningState = RunningState.STOPPED;
return (true);
}
@ -192,6 +196,28 @@ public class PollingAutomationExecutor
/*******************************************************************************
** Setter for qInstance
**
*******************************************************************************/
public void setQInstance(QInstance qInstance)
{
this.qInstance = qInstance;
}
/*******************************************************************************
** Setter for name
**
*******************************************************************************/
public void setName(String name)
{
this.name = name;
}
/*******************************************************************************
** Setter for sessionSupplier
**

26
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java → qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java
View File

@ -28,6 +28,7 @@ import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@ -39,6 +40,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.scheduler.StandardScheduledExecutor;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
@ -49,9 +51,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for PollingAutomationExecutor
** Unit test for StandardScheduledExecutor
*******************************************************************************/
class PollingAutomationExecutorTest
class StandardScheduledExecutorTest
{
/*******************************************************************************
@ -89,7 +91,7 @@ class PollingAutomationExecutorTest
////////////////////////////////////////////////
// have the polling executor run "for awhile" //
////////////////////////////////////////////////
runPollingAutomationExecutorForAwhile(qInstance);
runPollingAutomationExecutorForAwhile(qInstance, QSession::new);
/////////////////////////////////////////////////
// query for the records - assert their status //
@ -112,8 +114,6 @@ class PollingAutomationExecutorTest
/*******************************************************************************
**
*******************************************************************************/
@ -140,15 +140,14 @@ class PollingAutomationExecutorTest
));
new InsertAction().execute(insertInput);
String uuid = UUID.randomUUID().toString();
String uuid = UUID.randomUUID().toString();
QSession session = new QSession();
session.setIdReference(uuid);
PollingAutomationExecutor.getInstance().setSessionSupplier(() -> session);
////////////////////////////////////////////////
// have the polling executor run "for awhile" //
////////////////////////////////////////////////
runPollingAutomationExecutorForAwhile(qInstance);
runPollingAutomationExecutorForAwhile(qInstance, () -> session);
/////////////////////////////////////////////////////////////////////////////////////////////////////
// assert that the uuid we put in our session was present in the CaptureSessionIdAutomationHandler //
@ -183,12 +182,17 @@ class PollingAutomationExecutorTest
/*******************************************************************************
**
*******************************************************************************/
private void runPollingAutomationExecutorForAwhile(QInstance qInstance)
private void runPollingAutomationExecutorForAwhile(QInstance qInstance, Supplier<QSession> sessionSupplier)
{
PollingAutomationExecutor pollingAutomationExecutor = PollingAutomationExecutor.getInstance();
PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, TestUtils.POLLING_AUTOMATION, sessionSupplier);
StandardScheduledExecutor pollingAutomationExecutor = new StandardScheduledExecutor(pollingAutomationRunner);
pollingAutomationExecutor.setInitialDelayMillis(0);
pollingAutomationExecutor.setDelayMillis(100);
pollingAutomationExecutor.start(qInstance, TestUtils.POLLING_AUTOMATION);
pollingAutomationExecutor.setQInstance(qInstance);
pollingAutomationExecutor.setName(TestUtils.POLLING_AUTOMATION);
pollingAutomationExecutor.setSessionSupplier(sessionSupplier);
pollingAutomationExecutor.start();
SleepUtils.sleep(1, TimeUnit.SECONDS);
pollingAutomationExecutor.stop();
}

131
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManagerTest.java Normal file
View File

@ -0,0 +1,131 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.scheduler;
import java.util.List;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for ScheduleManager
*******************************************************************************/
class ScheduleManagerTest
{
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
void afterEach()
{
ScheduleManager.resetSingleton();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testStartAndStop()
{
QInstance qInstance = TestUtils.defineInstance();
ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance);
scheduleManager.start();
assertThat(scheduleManager.getExecutors()).isNotEmpty();
scheduleManager.stopAsync();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testScheduledProcess() throws QInstanceValidationException
{
QInstance qInstance = TestUtils.defineInstance();
new QInstanceValidator().validate(qInstance);
qInstance.getAutomationProviders().clear();
qInstance.getQueueProviders().clear();
qInstance.addProcess(
new QProcessMetaData()
.withName("testScheduledProcess")
.withSchedule(new QScheduleMetaData().withRepeatMillis(2).withInitialDelaySeconds(0))
.withStepList(List.of(new QBackendStepMetaData()
.withName("step")
.withCode(new QCodeReference(BasicStep.class))))
);
BasicStep.counter = 0;
ScheduleManager scheduleManager = ScheduleManager.getInstance(qInstance);
scheduleManager.setSessionSupplier(QSession::new);
scheduleManager.start();
SleepUtils.sleep(50, TimeUnit.MILLISECONDS);
scheduleManager.stopAsync();
System.out.println("Ran: " + BasicStep.counter + " times");
assertTrue(BasicStep.counter > 1, "Scheduled process should have ran at least twice");
}
/*******************************************************************************
**
*******************************************************************************/
public static class BasicStep implements BackendStep
{
static int counter = 0;
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
counter++;
}
}
}

47
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
@ -72,6 +73,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
@ -121,7 +125,8 @@ public class TestUtils
public static final String POSSIBLE_VALUE_SOURCE_CUSTOM = "custom"; // custom-type
public static final String POSSIBLE_VALUE_SOURCE_AUTOMATION_STATUS = "automationStatus";
public static final String POLLING_AUTOMATION = "polling";
public static final String POLLING_AUTOMATION = "polling";
public static final String DEFAULT_QUEUE_PROVIDER = "defaultQueueProvider";
@ -156,6 +161,9 @@ public class TestUtils
qInstance.addAutomationProvider(definePollingAutomationProvider());
qInstance.addQueueProvider(defineSqsProvider());
qInstance.addQueue(defineTestSqsQueue());
defineWidgets(qInstance);
defineApps(qInstance);
@ -841,4 +849,41 @@ public class TestUtils
return (new QPossibleValue<>(idValue, "Custom[" + idValue + "]"));
}
}
/*******************************************************************************
**
*******************************************************************************/
private static QQueueProviderMetaData defineSqsProvider()
{
QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
String accessKey = interpreter.interpret("${env.SQS_ACCESS_KEY}");
String secretKey = interpreter.interpret("${env.SQS_SECRET_KEY}");
String region = interpreter.interpret("${env.SQS_REGION}");
String baseURL = interpreter.interpret("${env.SQS_BASE_URL}");
return (new SQSQueueProviderMetaData()
.withName(DEFAULT_QUEUE_PROVIDER)
.withAccessKey(accessKey)
.withSecretKey(secretKey)
.withRegion(region)
.withBaseURL(baseURL));
}
/*******************************************************************************
**
*******************************************************************************/
private static QQueueMetaData defineTestSqsQueue()
{
return (new QQueueMetaData()
.withName("testSQSQueue")
.withProviderName(DEFAULT_QUEUE_PROVIDER)
.withQueueName("test-queue")
.withProcessName("receiveEasypostTrackerWebhook"));
}
}

11
qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/EasyPostApiTest.java
View File

@ -45,6 +45,15 @@ public class EasyPostApiTest
{
/*******************************************************************************
** Supported Tracking Numbers (and their statuses)
**
** EZ1000000001 : pre-transit
** EZ2000000002 : in transit
** EZ3000000003 : out for delivery
** EZ4000000004 : delivered
** EZ5000000005 : return to sender
** EZ6000000006 : failure
** EZ7000000007 : unknown
**
*******************************************************************************/
@Test
@ -53,7 +62,7 @@ public class EasyPostApiTest
QRecord record = new QRecord()
.withValue("__ignoreMe", "123")
.withValue("carrierCode", "USPS")
.withValue("trackingNo", "EZ1000000001");
.withValue("trackingNo", "EZ4000000004");
InsertInput insertInput = new InsertInput(TestUtils.defineInstance());
insertInput.setSession(new QSession());

2
qqq-backend-module-rdbms/pom.xml
View File

@ -102,7 +102,7 @@
</configuration>
<executions>
<execution>
<phase>package</phase>
<phase>${plugin.shade.phase}</phase>
<goals>
<goal>shade</goal>
</goals>

2
qqq-middleware-lambda/pom.xml
View File

@ -124,7 +124,7 @@
</configuration>
<executions>
<execution>
<phase>package</phase>
<phase>${plugin.shade.phase}</phase>
<goals>
<goal>shade</goal>
</goals>

23
qqq-utility-lambdas/README.md Normal file
View File

@ -0,0 +1,23 @@
# qqq-middleware-javalin
This is a utility module, for use with QQQ applications, that want to
have simple, related AWS lambda functions - which aren't actually running
all of QQQ - but are helpful for integrating qqq with lambdas.
## License
QQQ - Low-code Application Framework for Engineers. \
Copyright (C) 2022. Kingsrook, LLC \
651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States \
contact@kingsrook.com | https://github.com/Kingsrook/
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.

286
qqq-utility-lambdas/pom.xml Normal file
View File

@ -0,0 +1,286 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ QQQ - Low-code Application Framework for Engineers.
~ Copyright (C) 2021-2022. Kingsrook, LLC
~ 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
~ contact@kingsrook.com
~ https://github.com/Kingsrook/
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU Affero General Public License as
~ published by the Free Software Foundation, either version 3 of the
~ License, or (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU Affero General Public License for more details.
~
~ You should have received a copy of the GNU Affero General Public License
~ along with this program. If not, see <https://www.gnu.org/licenses/>.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>qqq-utility-lambdas</artifactId>
<parent>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-parent-project</artifactId>
<version>${revision}</version>
</parent>
<properties>
<!-- props specifically to this module -->
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
</properties>
<dependencies>
<!-- 3rd party deps specifically for this module -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-lambda-java-core</artifactId>
<version>1.2.1</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-sqs</artifactId>
<version>1.12.321</version>
</dependency>
</dependencies>
<build>
<plugins>
<!-- Common plugins for all qqq modules -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<compilerArgument>-Xlint:unchecked</compilerArgument>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M5</version>
<configuration>
<!-- Sets the VM argument line used when integration tests are run. -->
<argLine>@{jaCoCoArgLine}</argLine>
</configuration>
</plugin>
<!--
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>3.1.2</version>
<dependencies>
<dependency>
<groupId>com.puppycrawl.tools</groupId>
<artifactId>checkstyle</artifactId>
<version>9.0</version>
</dependency>
</dependencies>
<executions>
<execution>
<id>validate</id>
<phase>validate</phase>
<configuration>
<configLocation>checkstyle/config.xml</configLocation>
<headerLocation>checkstyle/license.txt</headerLocation>
<encoding>UTF-8</encoding>
<consoleOutput>true</consoleOutput>
<failsOnError>false</failsOnError>
<failOnViolation>true</failOnViolation>
<violationSeverity>warning</violationSeverity>
<excludes>**/target/generated-sources/*.*</excludes>
</configuration>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
-->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<version>1.1.0</version>
<configuration>
<updatePomFile>true</updatePomFile>
<flattenMode>resolveCiFriendliesOnly</flattenMode>
</configuration>
<executions>
<execution>
<id>flatten</id>
<phase>process-resources</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.amashchenko.maven.plugin</groupId>
<artifactId>gitflow-maven-plugin</artifactId>
<version>1.18.0</version>
<configuration>
<gitFlowConfig>
<productionBranch>main</productionBranch>
<developmentBranch>dev</developmentBranch>
<versionTagPrefix>version-</versionTagPrefix>
</gitFlowConfig>
<skipFeatureVersion>true</skipFeatureVersion> <!-- Keep feature names out of versions -->
<postReleaseGoals>install</postReleaseGoals> <!-- Let CI run deploys -->
<commitDevelopmentVersionAtStart>true</commitDevelopmentVersionAtStart>
<versionDigitToIncrement>1</versionDigitToIncrement> <!-- In general, we update the minor -->
<versionProperty>revision</versionProperty>
<skipUpdateVersion>true</skipUpdateVersion>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.8</version>
<executions>
<execution>
<id>pre-unit-test</id>
<goals>
<goal>prepare-agent</goal>
</goals>
<configuration>
<propertyName>jaCoCoArgLine</propertyName>
</configuration>
</execution>
<execution>
<id>unit-test-check</id>
<goals>
<goal>check</goal>
</goals>
<configuration>
<!-- Gives us the ability to pass a parameter to not fail due to coverage E.g. -Dcoverage.haltOnFailure=false -->
<haltOnFailure>${coverage.haltOnFailure}</haltOnFailure>
<rules>
<rule>
<element>BUNDLE</element>
<limits>
<limit>
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>${coverage.instructionCoveredRatioMinimum}</minimum>
</limit>
<limit>
<counter>CLASS</counter>
<value>COVEREDRATIO</value>
<minimum>${coverage.classCoveredRatioMinimum}</minimum>
</limit>
</limits>
</rule>
</rules>
</configuration>
</execution>
<execution>
<id>post-unit-test</id>
<phase>verify</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>exec-maven-plugin</artifactId>
<groupId>org.codehaus.mojo</groupId>
<version>3.0.0</version>
<executions>
<execution>
<id>test-coverage-summary</id>
<phase>verify</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>sh</executable>
<arguments>
<argument>-c</argument>
<argument>
<![CDATA[
if [ ! -e target/site/jacoco/index.html ]; then
echo "No jacoco coverage report here.";
exit;
fi
echo
echo "Jacoco coverage summary report:"
echo " See also target/site/jacoco/index.html"
echo " and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
echo "------------------------------------------------------------"
which xpath > /dev/null 2>&1
if [ "$?" == "0" ]; then
echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
rm /tmp/$$.headers /tmp/$$.values
else
echo "xpath is not installed. Jacoco coverage summary will not be produced here..";
fi
]]>
</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>github-qqq-maven-registry</id>
<name>GitHub QQQ Maven Registry</name>
<url>https://maven.pkg.github.com/Kingsrook/qqq-maven-registry</url>
</repository>
</repositories>
<distributionManagement>
<repository>
<id>github-qqq-maven-registry</id>
<name>GitHub QQQ Maven Registry</name>
<url>https://maven.pkg.github.com/Kingsrook/qqq-maven-registry</url>
</repository>
</distributionManagement>
</project>

128
qqq-utility-lambdas/src/main/java/com/kingsrook/qqq/utilitylambdas/QPostToSQSLambda.java Normal file
View File

@ -0,0 +1,128 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.utilitylambdas;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
import com.amazonaws.services.sqs.model.SendMessageRequest;
/*******************************************************************************
** QQQ Utility Lambda to post input data to SQS.
**
** Requires environment variable: QUEUE_URL (e.g., https://sqs.us-east-0.amazonaws.com/111122223333/my-queue-name)
*******************************************************************************/
public class QPostToSQSLambda implements RequestStreamHandler
{
protected Context context;
/*******************************************************************************
** Entrypoint from AWS Lambda.
*******************************************************************************/
public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException
{
this.context = context;
try
{
String input = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
log("Full Input: " + input);
final AmazonSQS sqs = AmazonSQSClientBuilder.defaultClient();
SendMessageRequest sendMessageRequest = new SendMessageRequest()
.withQueueUrl(System.getenv("QUEUE_URL"))
.withMessageBody(input);
sqs.sendMessage(sendMessageRequest);
writeResponse(outputStream, "OK");
}
catch(Exception e)
{
log(e);
throw (new IOException(e));
// writeResponse(outputStream, requestId, 500, "Uncaught error handing request: " + e.getMessage());
}
}
/*******************************************************************************
** Write to the cloudwatch logs.
*******************************************************************************/
protected void log(String message)
{
if(message == null)
{
return;
}
context.getLogger().log(message + (message.endsWith("\n") ? "" : "\n"));
}
/*******************************************************************************
**
*******************************************************************************/
protected void log(Throwable t)
{
if(t == null)
{
return;
}
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
t.printStackTrace(pw);
log(sw + "\n");
}
/*******************************************************************************
**
*******************************************************************************/
protected void writeResponse(OutputStream outputStream, String messageBody) throws IOException
{
StringBuilder body = new StringBuilder("{");
if(messageBody != null && !messageBody.equals(""))
{
body.append("\"body\":\"").append(messageBody.replaceAll("\"", "'")).append("\"");
}
body.append("}");
outputStream.write(body.toString().getBytes(StandardCharsets.UTF_8));
}
}