QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add option to break sqsPoller loop after a given number of iterations - and to config this, plus maxNumberOfMessages and waitTime on a per-queue and/or per-provider level

Browse Source
This commit is contained in:
darin.kelkhoff
2024-06-20 16:44:14 -05:00
parent c748977a1b
commit 9d24e61949
5 changed files with 360 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/queues/SQSQueuePoller.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.queues;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.function.Supplier;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
@ -41,6 +42,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSPollerSettings;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
@ -90,15 +93,17 @@ public class SQSQueuePoller implements Runnable
}
queueUrl += queueMetaData.getQueueName();
while(true)
SQSPollerSettings sqsPollerSettings = getSqsPollerSettings(queueProviderMetaData, queueMetaData);
for(int loop = 0; loop < sqsPollerSettings.getMaxLoops(); loop++)
{
///////////////////////////////
// fetch a batch of messages //
///////////////////////////////
ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest();
receiveMessageRequest.setQueueUrl(queueUrl);
receiveMessageRequest.setMaxNumberOfMessages(10);
receiveMessageRequest.setWaitTimeSeconds(20); // help urge SQS to query multiple servers and find more messages
receiveMessageRequest.setMaxNumberOfMessages(sqsPollerSettings.getMaxNumberOfMessages());
receiveMessageRequest.setWaitTimeSeconds(sqsPollerSettings.getWaitTimeSeconds()); // larger value (e.g., 20) can help urge SQS to query multiple servers and find more messages
ReceiveMessageResult receiveMessageResult = sqs.receiveMessage(receiveMessageRequest);
if(receiveMessageResult.getMessages().isEmpty())
{
@ -177,6 +182,47 @@ public class SQSQueuePoller implements Runnable
/*******************************************************************************
** For a given queueProvider and queue, get the poller settings to use (using
** default values if none are set at either level).
*******************************************************************************/
static SQSPollerSettings getSqsPollerSettings(SQSQueueProviderMetaData queueProviderMetaData, QQueueMetaData queueMetaData)
{
/////////////////////////////////
// start with default settings //
/////////////////////////////////
SQSPollerSettings sqsPollerSettings = new SQSPollerSettings()
.withMaxLoops(Integer.MAX_VALUE)
.withMaxNumberOfMessages(10)
.withWaitTimeSeconds(20);
/////////////////////////////////////////////////////////////////////
// if the queue provider has settings, let them overwrite defaults //
/////////////////////////////////////////////////////////////////////
if(queueProviderMetaData != null && queueProviderMetaData.getPollerSettings() != null)
{
SQSPollerSettings providerSettings = queueProviderMetaData.getPollerSettings();
sqsPollerSettings.setMaxLoops(Objects.requireNonNullElse(providerSettings.getMaxLoops(), sqsPollerSettings.getMaxLoops()));
sqsPollerSettings.setMaxNumberOfMessages(Objects.requireNonNullElse(providerSettings.getMaxNumberOfMessages(), sqsPollerSettings.getMaxNumberOfMessages()));
sqsPollerSettings.setWaitTimeSeconds(Objects.requireNonNullElse(providerSettings.getWaitTimeSeconds(), sqsPollerSettings.getWaitTimeSeconds()));
}
////////////////////////////////////////////////////////////
// if the queue has settings, let them overwrite defaults //
////////////////////////////////////////////////////////////
if(queueMetaData instanceof SQSQueueMetaData sqsQueueMetaData && sqsQueueMetaData.getPollerSettings() != null)
{
SQSPollerSettings providerSettings = sqsQueueMetaData.getPollerSettings();
sqsPollerSettings.setMaxLoops(Objects.requireNonNullElse(providerSettings.getMaxLoops(), sqsPollerSettings.getMaxLoops()));
sqsPollerSettings.setMaxNumberOfMessages(Objects.requireNonNullElse(providerSettings.getMaxNumberOfMessages(), sqsPollerSettings.getMaxNumberOfMessages()));
sqsPollerSettings.setWaitTimeSeconds(Objects.requireNonNullElse(providerSettings.getWaitTimeSeconds(), sqsPollerSettings.getWaitTimeSeconds()));
}
return sqsPollerSettings;
}
/*******************************************************************************
** Setter for queueProviderMetaData
**

128
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSPollerSettings.java Normal file
View File

@ -0,0 +1,128 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.queues;
/*******************************************************************************
** settings that can be applied to either an SQSQueue or an SQSQueueProvider,
** to control what the SQSQueuePoller does when it receives from AWS.
*******************************************************************************/
public class SQSPollerSettings
{
private Integer maxNumberOfMessages;
private Integer waitTimeSeconds;
private Integer maxLoops;
/*******************************************************************************
** Getter for maxNumberOfMessages
*******************************************************************************/
public Integer getMaxNumberOfMessages()
{
return (this.maxNumberOfMessages);
}
/*******************************************************************************
** Setter for maxNumberOfMessages
*******************************************************************************/
public void setMaxNumberOfMessages(Integer maxNumberOfMessages)
{
this.maxNumberOfMessages = maxNumberOfMessages;
}
/*******************************************************************************
** Fluent setter for maxNumberOfMessages
*******************************************************************************/
public SQSPollerSettings withMaxNumberOfMessages(Integer maxNumberOfMessages)
{
this.maxNumberOfMessages = maxNumberOfMessages;
return (this);
}
/*******************************************************************************
** Getter for waitTimeSeconds
*******************************************************************************/
public Integer getWaitTimeSeconds()
{
return (this.waitTimeSeconds);
}
/*******************************************************************************
** Setter for waitTimeSeconds
*******************************************************************************/
public void setWaitTimeSeconds(Integer waitTimeSeconds)
{
this.waitTimeSeconds = waitTimeSeconds;
}
/*******************************************************************************
** Fluent setter for waitTimeSeconds
*******************************************************************************/
public SQSPollerSettings withWaitTimeSeconds(Integer waitTimeSeconds)
{
this.waitTimeSeconds = waitTimeSeconds;
return (this);
}
/*******************************************************************************
** Getter for maxLoops
*******************************************************************************/
public Integer getMaxLoops()
{
return (this.maxLoops);
}
/*******************************************************************************
** Setter for maxLoops
*******************************************************************************/
public void setMaxLoops(Integer maxLoops)
{
this.maxLoops = maxLoops;
}
/*******************************************************************************
** Fluent setter for maxLoops
*******************************************************************************/
public SQSPollerSettings withMaxLoops(Integer maxLoops)
{
this.maxLoops = maxLoops;
return (this);
}
}

63
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueMetaData.java Normal file
View File

@ -0,0 +1,63 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.queues;
/*******************************************************************************
** SQS subclass of meta-data for a specific Queue
*******************************************************************************/
public class SQSQueueMetaData extends QQueueMetaData
{
private SQSPollerSettings pollerSettings;
/*******************************************************************************
** Getter for pollerSettings
*******************************************************************************/
public SQSPollerSettings getPollerSettings()
{
return (this.pollerSettings);
}
/*******************************************************************************
** Setter for pollerSettings
*******************************************************************************/
public void setPollerSettings(SQSPollerSettings pollerSettings)
{
this.pollerSettings = pollerSettings;
}
/*******************************************************************************
** Fluent setter for pollerSettings
*******************************************************************************/
public SQSQueueMetaData withPollerSettings(SQSPollerSettings pollerSettings)
{
this.pollerSettings = pollerSettings;
return (this);
}
}

33
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/SQSQueueProviderMetaData.java
View File

@ -36,6 +36,8 @@ public class SQSQueueProviderMetaData extends QQueueProviderMetaData
private String region;
private String baseURL;
private SQSPollerSettings pollerSettings;
/*******************************************************************************
@ -196,4 +198,35 @@ public class SQSQueueProviderMetaData extends QQueueProviderMetaData
return (this);
}
/*******************************************************************************
** Getter for pollerSettings
*******************************************************************************/
public SQSPollerSettings getPollerSettings()
{
return (this.pollerSettings);
}
/*******************************************************************************
** Setter for pollerSettings
*******************************************************************************/
public void setPollerSettings(SQSPollerSettings pollerSettings)
{
this.pollerSettings = pollerSettings;
}
/*******************************************************************************
** Fluent setter for pollerSettings
*******************************************************************************/
public SQSQueueProviderMetaData withPollerSettings(SQSPollerSettings pollerSettings)
{
this.pollerSettings = pollerSettings;
return (this);
}
}

87
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/queues/SQSQueuePollerTest.java Normal file
View File

@ -0,0 +1,87 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.queues;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSPollerSettings;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for SQSQueuePoller
*******************************************************************************/
class SQSQueuePollerTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetSqsPollerSettings()
{
///////////////////
// defaults only //
///////////////////
assertSettings(Integer.MAX_VALUE, 10, 20, SQSQueuePoller.getSqsPollerSettings(null, null));
assertSettings(Integer.MAX_VALUE, 10, 20, SQSQueuePoller.getSqsPollerSettings(new SQSQueueProviderMetaData(), new SQSQueueMetaData()));
assertSettings(Integer.MAX_VALUE, 10, 20, SQSQueuePoller.getSqsPollerSettings(new SQSQueueProviderMetaData().withPollerSettings(new SQSPollerSettings()), new SQSQueueMetaData().withPollerSettings(new SQSPollerSettings())));
///////////////////////////////////
// settings only in the provider //
///////////////////////////////////
assertSettings(100, 5, 1, SQSQueuePoller.getSqsPollerSettings(
new SQSQueueProviderMetaData().withPollerSettings(new SQSPollerSettings().withMaxLoops(100).withMaxNumberOfMessages(5).withWaitTimeSeconds(1)),
new QQueueMetaData()));
////////////////////////////////
// settings only in the queue //
////////////////////////////////
assertSettings(90, 4, 2, SQSQueuePoller.getSqsPollerSettings(
new SQSQueueProviderMetaData(),
new SQSQueueMetaData().withPollerSettings(new SQSPollerSettings().withMaxLoops(90).withMaxNumberOfMessages(4).withWaitTimeSeconds(2))));
/////////////////////////////////////////
// mix of default, provider, and queue //
/////////////////////////////////////////
assertSettings(Integer.MAX_VALUE, 5, 2, SQSQueuePoller.getSqsPollerSettings(
new SQSQueueProviderMetaData().withPollerSettings(new SQSPollerSettings().withMaxNumberOfMessages(5)),
new SQSQueueMetaData().withPollerSettings(new SQSPollerSettings().withWaitTimeSeconds(2))));
}
/*******************************************************************************
**
*******************************************************************************/
private void assertSettings(Integer maxLoops, Integer maxNumberOfMessages, Integer waitTimeSeconds, SQSPollerSettings sqsPollerSettings)
{
assertEquals(maxLoops, sqsPollerSettings.getMaxLoops());
assertEquals(maxNumberOfMessages, sqsPollerSettings.getMaxNumberOfMessages());
assertEquals(waitTimeSeconds, sqsPollerSettings.getWaitTimeSeconds());
}
}