QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

updated to allow 'trying again' when server side 500 error occur in makeRequest()

Browse Source
This commit is contained in:
Tim Chamberlain
2023-05-25 11:34:50 -05:00
parent 5185758855
commit 489f12996d
3 changed files with 151 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java
View File

@ -63,6 +63,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.module.api.exceptions.OAuthCredentialsException;
import com.kingsrook.qqq.backend.module.api.exceptions.OAuthExpiredTokenException;
import com.kingsrook.qqq.backend.module.api.exceptions.RateLimitException;
import com.kingsrook.qqq.backend.module.api.exceptions.RetryableServerErrorException;
import com.kingsrook.qqq.backend.module.api.model.AuthorizationType;
import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog;
import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendMetaData;
@ -878,8 +879,10 @@ public class BaseAPIActionUtil
*******************************************************************************/
public QHttpResponse makeRequest(QTableMetaData table, HttpRequestBase request) throws QException
{
int sleepMillis = getInitialRateLimitBackoffMillis();
int rateLimitSleepMillis = getInitialRateLimitBackoffMillis();
int serverErrorsSleepMillis = getInitialServerErrorBackoffMillis();
int rateLimitsCaught = 0;
int serverErrorsCaught = 0;
boolean caughtAnOAuthExpiredToken = false;
while(true)
@ -913,7 +916,11 @@ public class BaseAPIActionUtil
{
throw (new RateLimitException(qResponse.getContent()));
}
if(statusCode >= 400)
else if(shouldBeRetryableServerErrorException(qResponse))
{
throw (new RetryableServerErrorException(qResponse.getContent()));
}
else if(statusCode >= 400)
{
handleResponseError(table, request, qResponse);
}
@ -950,9 +957,22 @@ public class BaseAPIActionUtil
throw (new QException(rle));
}
LOG.info("Caught RateLimitException", logPair("rateLimitsCaught", rateLimitsCaught), logPair("uri", request.getURI()), logPair("table", table.getName()), logPair("sleeping", sleepMillis));
SleepUtils.sleep(sleepMillis, TimeUnit.MILLISECONDS);
sleepMillis *= 2;
LOG.info("Caught RateLimitException", logPair("rateLimitsCaught", rateLimitsCaught), logPair("uri", request.getURI()), logPair("table", table.getName()), logPair("sleeping", rateLimitSleepMillis));
SleepUtils.sleep(rateLimitSleepMillis, TimeUnit.MILLISECONDS);
rateLimitSleepMillis *= 2;
}
catch(RetryableServerErrorException see)
{
serverErrorsCaught++;
if(serverErrorsCaught > getMaxAllowedServerErrors())
{
LOG.error("Giving up " + request.getMethod() + " to [" + table.getName() + "] after too many server-side errors (" + getMaxAllowedServerErrors() + ")");
throw (new QException(see));
}
LOG.info("Caught Server-side error during API request", logPair("serverErrorsCaught", serverErrorsCaught), logPair("uri", request.getURI()), logPair("table", table.getName()), logPair("sleeping", serverErrorsSleepMillis));
SleepUtils.sleep(serverErrorsSleepMillis, TimeUnit.MILLISECONDS);
serverErrorsSleepMillis *= 2;
}
catch(QException qe)
{
@ -972,6 +992,16 @@ public class BaseAPIActionUtil
/*******************************************************************************
**
*******************************************************************************/
private boolean shouldBeRetryableServerErrorException(QHttpResponse qResponse)
{
return (qResponse.getStatusCode() != null && qResponse.getStatusCode() >= 500);
}
/*******************************************************************************
** one-line method, factored out so mock/tests can override
*******************************************************************************/
@ -1141,6 +1171,16 @@ public class BaseAPIActionUtil
/*******************************************************************************
**
*******************************************************************************/
protected int getInitialServerErrorBackoffMillis()
{
return (500);
}
/*******************************************************************************
**
*******************************************************************************/
@ -1151,6 +1191,16 @@ public class BaseAPIActionUtil
/*******************************************************************************
**
*******************************************************************************/
protected int getMaxAllowedServerErrors()
{
return (3);
}
/*******************************************************************************
**
*******************************************************************************/

42
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/RetryableServerErrorException.java Normal file
View File

@ -0,0 +1,42 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.api.exceptions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
/*******************************************************************************
**
*******************************************************************************/
public class RetryableServerErrorException extends QException
{
/*******************************************************************************
**
*******************************************************************************/
public RetryableServerErrorException(String message)
{
super(message);
}
}

65
qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtilTest.java
View File

@ -124,9 +124,16 @@ class BaseAPIActionUtilTest extends BaseTest
// avoid the fully mocked makeRequest //
////////////////////////////////////////
mockApiUtilsHelper.setUseMock(false);
mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
{"error": "Server error"}
"""));
//////////////////////////
// set to retry 3 times //
//////////////////////////
for(int i = 0; i < 4; i++)
{
mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
{"error": "Server error"}
"""));
}
CountInput countInput = new CountInput();
countInput.setTableName(TestUtils.MOCK_TABLE_NAME);
@ -290,9 +297,16 @@ class BaseAPIActionUtilTest extends BaseTest
// avoid the fully mocked makeRequest //
////////////////////////////////////////
mockApiUtilsHelper.setUseMock(false);
mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
{"error": "Server error"}
"""));
//////////////////////////
// set to retry 3 times //
//////////////////////////
for(int i = 0; i < 4; i++)
{
mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
{"error": "Server error"}
"""));
}
QueryInput queryInput = new QueryInput();
queryInput.setTableName(TestUtils.MOCK_TABLE_NAME);
@ -344,9 +358,16 @@ class BaseAPIActionUtilTest extends BaseTest
// avoid the fully mocked makeRequest //
////////////////////////////////////////
mockApiUtilsHelper.setUseMock(false);
mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
{"error": "Server error"}
"""));
//////////////////////////
// set to retry 3 times //
//////////////////////////
for(int i = 0; i < 4; i++)
{
mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
{"error": "Server error"}
"""));
}
InsertInput insertInput = new InsertInput();
insertInput.setRecords(List.of(new QRecord().withValue("name", "Milhouse")));
@ -411,9 +432,13 @@ class BaseAPIActionUtilTest extends BaseTest
// avoid the fully mocked makeRequest //
////////////////////////////////////////
mockApiUtilsHelper.setUseMock(false);
mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
{"error": "Server error"}
"""));
for(int i = 0; i < 4; i++)
{
mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
{"error": "Server error"}
"""));
}
UpdateInput updateInput = new UpdateInput();
updateInput.setRecords(List.of(new QRecord().withValue("name", "Milhouse")));
@ -682,4 +707,20 @@ class BaseAPIActionUtilTest extends BaseTest
return (new GetAction().execute(getInput));
}
/*******************************************************************************
** subclass of base api action utils that can be used to test overriding methods
*******************************************************************************/
private class BaseAPIActionUtilSubclass extends BaseAPIActionUtil
{
/*******************************************************************************
**
*******************************************************************************/
private boolean shouldBeRetryableServerErrorException(QHttpResponse qResponse)
{
return (false);
}
}
}