QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #100 from Kingsrook/feature/sqs-max-loops

Browse Source 
Feature/sqs max loops
This commit is contained in:
darin.kelkhoff
2024-06-21 08:29:06 -05:00
committed by GitHub
parent c748977a1b e8978a7f92
commit 848353d804
6 changed files with 402 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/queues/SQSQueuePoller.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.queues;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.function.Supplier;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
@ -41,6 +42,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSPollerSettings;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
@ -90,15 +93,17 @@ public class SQSQueuePoller implements Runnable
}
queueUrl += queueMetaData.getQueueName();
while(true)
SQSPollerSettings sqsPollerSettings = getSqsPollerSettings(queueProviderMetaData, queueMetaData);
for(int loop = 0; loop < sqsPollerSettings.getMaxLoops(); loop++)
{
///////////////////////////////
// fetch a batch of messages //
///////////////////////////////
ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest();
receiveMessageRequest.setQueueUrl(queueUrl);
receiveMessageRequest.setMaxNumberOfMessages(10);
receiveMessageRequest.setWaitTimeSeconds(20); // help urge SQS to query multiple servers and find more messages
receiveMessageRequest.setMaxNumberOfMessages(sqsPollerSettings.getMaxNumberOfMessages());
receiveMessageRequest.setWaitTimeSeconds(sqsPollerSettings.getWaitTimeSeconds()); // larger value (e.g., 20) can help urge SQS to query multiple servers and find more messages
ReceiveMessageResult receiveMessageResult = sqs.receiveMessage(receiveMessageRequest);
if(receiveMessageResult.getMessages().isEmpty())
{
@ -177,6 +182,47 @@ public class SQSQueuePoller implements Runnable
/*******************************************************************************
** For a given queueProvider and queue, get the poller settings to use (using
** default values if none are set at either level).
*******************************************************************************/
static SQSPollerSettings getSqsPollerSettings(SQSQueueProviderMetaData queueProviderMetaData, QQueueMetaData queueMetaData)
{
/////////////////////////////////
// start with default settings //
/////////////////////////////////
SQSPollerSettings sqsPollerSettings = new SQSPollerSettings()
.withMaxLoops(Integer.MAX_VALUE)
.withMaxNumberOfMessages(10)
.withWaitTimeSeconds(20);
/////////////////////////////////////////////////////////////////////
// if the queue provider has settings, let them overwrite defaults //
/////////////////////////////////////////////////////////////////////
if(queueProviderMetaData != null && queueProviderMetaData.getPollerSettings() != null)
{
SQSPollerSettings providerSettings = queueProviderMetaData.getPollerSettings();
sqsPollerSettings.setMaxLoops(Objects.requireNonNullElse(providerSettings.getMaxLoops(), sqsPollerSettings.getMaxLoops()));
sqsPollerSettings.setMaxNumberOfMessages(Objects.requireNonNullElse(providerSettings.getMaxNumberOfMessages(), sqsPollerSettings.getMaxNumberOfMessages()));
sqsPollerSettings.setWaitTimeSeconds(Objects.requireNonNullElse(providerSettings.getWaitTimeSeconds(), sqsPollerSettings.getWaitTimeSeconds()));
}
////////////////////////////////////////////////////////////
// if the queue has settings, let them overwrite defaults //
////////////////////////////////////////////////////////////
if(queueMetaData instanceof SQSQueueMetaData sqsQueueMetaData && sqsQueueMetaData.getPollerSettings() != null)
{
SQSPollerSettings providerSettings = sqsQueueMetaData.getPollerSettings();
sqsPollerSettings.setMaxLoops(Objects.requireNonNullElse(providerSettings.getMaxLoops(), sqsPollerSettings.getMaxLoops()));
sqsPollerSettings.setMaxNumberOfMessages(Objects.requireNonNullElse(providerSettings.getMaxNumberOfMessages(), sqsPollerSettings.getMaxNumberOfMessages()));
sqsPollerSettings.setWaitTimeSeconds(Objects.requireNonNullElse(providerSettings.getWaitTimeSeconds(), sqsPollerSettings.getWaitTimeSeconds()));
}
return sqsPollerSettings;
}
/*******************************************************************************
** Setter for queueProviderMetaData
**

43
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
View File

@ -79,6 +79,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QSupplementalProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QueueType;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
@ -439,11 +441,30 @@ public class QInstanceValidator
if(queueProvider instanceof SQSQueueProviderMetaData sqsQueueProvider)
{
if(queueProvider.getType() != null)
{
assertCondition(queueProvider.getType().equals(QueueType.SQS), "Inconsistent Type/class given for queueProvider: " + name + " (SQSQueueProviderMetaData is not allowed for type " + queueProvider.getType() + ")");
}
assertCondition(StringUtils.hasContent(sqsQueueProvider.getAccessKey()), "Missing accessKey for SQSQueueProvider: " + name);
assertCondition(StringUtils.hasContent(sqsQueueProvider.getSecretKey()), "Missing secretKey for SQSQueueProvider: " + name);
assertCondition(StringUtils.hasContent(sqsQueueProvider.getBaseURL()), "Missing baseURL for SQSQueueProvider: " + name);
assertCondition(StringUtils.hasContent(sqsQueueProvider.getRegion()), "Missing region for SQSQueueProvider: " + name);
}
else if(queueProvider.getClass().equals(QQueueProviderMetaData.class))
{
/////////////////////////////////////////////////////////////////////
// this just means a subtype wasn't used, so, it should be allowed //
// (unless we had a case where a type required a subtype?) //
/////////////////////////////////////////////////////////////////////
}
else
{
if(queueProvider.getType() != null)
{
assertCondition(!queueProvider.getType().equals(QueueType.SQS), "Inconsistent Type/class given for queueProvider: " + name + " (" + queueProvider.getClass().getSimpleName() + " is not allowed for type " + queueProvider.getType() + ")");
}
}
runPlugins(QQueueProviderMetaData.class, queueProvider, qInstance);
});
@ -454,7 +475,27 @@ public class QInstanceValidator
qInstance.getQueues().forEach((name, queue) ->
{
assertCondition(Objects.equals(name, queue.getName()), "Inconsistent naming for queue: " + name + "/" + queue.getName() + ".");
assertCondition(qInstance.getQueueProvider(queue.getProviderName()) != null, "Unrecognized queue providerName for queue: " + name);
QQueueProviderMetaData queueProvider = qInstance.getQueueProvider(queue.getProviderName());
if(assertCondition(queueProvider != null, "Unrecognized queue providerName for queue: " + name))
{
if(queue instanceof SQSQueueMetaData)
{
assertCondition(queueProvider.getType().equals(QueueType.SQS), "Inconsistent class given for queueMetaData: " + name + " (SQSQueueMetaData is not allowed for queue provider of type " + queueProvider.getType() + ")");
}
else if(queue.getClass().equals(QQueueMetaData.class))
{
////////////////////////////////////////////////////////////////////
// this just means a subtype wasn't used, so, it should be //
// allowed (unless we had a case where a type required a subtype? //
////////////////////////////////////////////////////////////////////
}
else
{
assertCondition(!queueProvider.getType().equals(QueueType.SQS), "Inconsistent class given for queueProvider: " + name + " (" + queue.getClass().getSimpleName() + " is not allowed for type " + queueProvider.getType() + ")");
}
}
assertCondition(StringUtils.hasContent(queue.getQueueName()), "Missing queueName for queue: " + name);
if(assertCondition(StringUtils.hasContent(queue.getProcessName()), "Missing processName for queue: " + name))
{

128
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSPollerSettings.java Normal file
View File

@ -0,0 +1,128 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.queues;
/*******************************************************************************
** settings that can be applied to either an SQSQueue or an SQSQueueProvider,
** to control what the SQSQueuePoller does when it receives from AWS.
*******************************************************************************/
public class SQSPollerSettings
{
private Integer maxNumberOfMessages;
private Integer waitTimeSeconds;
private Integer maxLoops;
/*******************************************************************************
** Getter for maxNumberOfMessages
*******************************************************************************/
public Integer getMaxNumberOfMessages()
{
return (this.maxNumberOfMessages);
}
/*******************************************************************************
** Setter for maxNumberOfMessages
*******************************************************************************/
public void setMaxNumberOfMessages(Integer maxNumberOfMessages)
{
this.maxNumberOfMessages = maxNumberOfMessages;
}
/*******************************************************************************
** Fluent setter for maxNumberOfMessages
*******************************************************************************/
public SQSPollerSettings withMaxNumberOfMessages(Integer maxNumberOfMessages)
{
this.maxNumberOfMessages = maxNumberOfMessages;
return (this);
}
/*******************************************************************************
** Getter for waitTimeSeconds
*******************************************************************************/
public Integer getWaitTimeSeconds()
{
return (this.waitTimeSeconds);
}
/*******************************************************************************
** Setter for waitTimeSeconds
*******************************************************************************/
public void setWaitTimeSeconds(Integer waitTimeSeconds)
{
this.waitTimeSeconds = waitTimeSeconds;
}
/*******************************************************************************
** Fluent setter for waitTimeSeconds
*******************************************************************************/
public SQSPollerSettings withWaitTimeSeconds(Integer waitTimeSeconds)
{
this.waitTimeSeconds = waitTimeSeconds;
return (this);
}
/*******************************************************************************
** Getter for maxLoops
*******************************************************************************/
public Integer getMaxLoops()
{
return (this.maxLoops);
}
/*******************************************************************************
** Setter for maxLoops
*******************************************************************************/
public void setMaxLoops(Integer maxLoops)
{
this.maxLoops = maxLoops;
}
/*******************************************************************************
** Fluent setter for maxLoops
*******************************************************************************/
public SQSPollerSettings withMaxLoops(Integer maxLoops)
{
this.maxLoops = maxLoops;
return (this);
}
}

63
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueMetaData.java Normal file
View File

@ -0,0 +1,63 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.queues;
/*******************************************************************************
** SQS subclass of meta-data for a specific Queue
*******************************************************************************/
public class SQSQueueMetaData extends QQueueMetaData
{
private SQSPollerSettings pollerSettings;
/*******************************************************************************
** Getter for pollerSettings
*******************************************************************************/
public SQSPollerSettings getPollerSettings()
{
return (this.pollerSettings);
}
/*******************************************************************************
** Setter for pollerSettings
*******************************************************************************/
public void setPollerSettings(SQSPollerSettings pollerSettings)
{
this.pollerSettings = pollerSettings;
}
/*******************************************************************************
** Fluent setter for pollerSettings
*******************************************************************************/
public SQSQueueMetaData withPollerSettings(SQSPollerSettings pollerSettings)
{
this.pollerSettings = pollerSettings;
return (this);
}
}

33
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java
View File

@ -36,6 +36,8 @@ public class SQSQueueProviderMetaData extends QQueueProviderMetaData
private String region;
private String baseURL;
private SQSPollerSettings pollerSettings;
/*******************************************************************************
@ -196,4 +198,35 @@ public class SQSQueueProviderMetaData extends QQueueProviderMetaData
return (this);
}
/*******************************************************************************
** Getter for pollerSettings
*******************************************************************************/
public SQSPollerSettings getPollerSettings()
{
return (this.pollerSettings);
}
/*******************************************************************************
** Setter for pollerSettings
*******************************************************************************/
public void setPollerSettings(SQSPollerSettings pollerSettings)
{
this.pollerSettings = pollerSettings;
}
/*******************************************************************************
** Fluent setter for pollerSettings
*******************************************************************************/
public SQSQueueProviderMetaData withPollerSettings(SQSPollerSettings pollerSettings)
{
this.pollerSettings = pollerSettings;
return (this);
}
}

87
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/queues/SQSQueuePollerTest.java Normal file
View File

@ -0,0 +1,87 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.queues;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSPollerSettings;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for SQSQueuePoller
*******************************************************************************/
class SQSQueuePollerTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetSqsPollerSettings()
{
///////////////////
// defaults only //
///////////////////
assertSettings(Integer.MAX_VALUE, 10, 20, SQSQueuePoller.getSqsPollerSettings(null, null));
assertSettings(Integer.MAX_VALUE, 10, 20, SQSQueuePoller.getSqsPollerSettings(new SQSQueueProviderMetaData(), new SQSQueueMetaData()));
assertSettings(Integer.MAX_VALUE, 10, 20, SQSQueuePoller.getSqsPollerSettings(new SQSQueueProviderMetaData().withPollerSettings(new SQSPollerSettings()), new SQSQueueMetaData().withPollerSettings(new SQSPollerSettings())));
///////////////////////////////////
// settings only in the provider //
///////////////////////////////////
assertSettings(100, 5, 1, SQSQueuePoller.getSqsPollerSettings(
new SQSQueueProviderMetaData().withPollerSettings(new SQSPollerSettings().withMaxLoops(100).withMaxNumberOfMessages(5).withWaitTimeSeconds(1)),
new QQueueMetaData()));
////////////////////////////////
// settings only in the queue //
////////////////////////////////
assertSettings(90, 4, 2, SQSQueuePoller.getSqsPollerSettings(
new SQSQueueProviderMetaData(),
new SQSQueueMetaData().withPollerSettings(new SQSPollerSettings().withMaxLoops(90).withMaxNumberOfMessages(4).withWaitTimeSeconds(2))));
/////////////////////////////////////////
// mix of default, provider, and queue //
/////////////////////////////////////////
assertSettings(Integer.MAX_VALUE, 5, 2, SQSQueuePoller.getSqsPollerSettings(
new SQSQueueProviderMetaData().withPollerSettings(new SQSPollerSettings().withMaxNumberOfMessages(5)),
new SQSQueueMetaData().withPollerSettings(new SQSPollerSettings().withWaitTimeSeconds(2))));
}
/*******************************************************************************
**
*******************************************************************************/
private void assertSettings(Integer maxLoops, Integer maxNumberOfMessages, Integer waitTimeSeconds, SQSPollerSettings sqsPollerSettings)
{
assertEquals(maxLoops, sqsPollerSettings.getMaxLoops());
assertEquals(maxNumberOfMessages, sqsPollerSettings.getMaxNumberOfMessages());
assertEquals(waitTimeSeconds, sqsPollerSettings.getWaitTimeSeconds());
}
}