diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 7b8769e7..7a3c46af 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -63,6 +63,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.module.api.exceptions.OAuthCredentialsException; import com.kingsrook.qqq.backend.module.api.exceptions.OAuthExpiredTokenException; import com.kingsrook.qqq.backend.module.api.exceptions.RateLimitException; +import com.kingsrook.qqq.backend.module.api.exceptions.RetryableServerErrorException; import com.kingsrook.qqq.backend.module.api.model.AuthorizationType; import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog; import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendMetaData; @@ -878,8 +879,10 @@ public class BaseAPIActionUtil *******************************************************************************/ public QHttpResponse makeRequest(QTableMetaData table, HttpRequestBase request) throws QException { - int sleepMillis = getInitialRateLimitBackoffMillis(); + int rateLimitSleepMillis = getInitialRateLimitBackoffMillis(); + int serverErrorsSleepMillis = getInitialServerErrorBackoffMillis(); int rateLimitsCaught = 0; + int serverErrorsCaught = 0; boolean caughtAnOAuthExpiredToken = false; while(true) @@ -913,7 +916,11 @@ public class BaseAPIActionUtil { throw (new RateLimitException(qResponse.getContent())); } - if(statusCode >= 400) + else if(shouldBeRetryableServerErrorException(qResponse)) + { + throw (new RetryableServerErrorException(qResponse.getContent())); + } + else if(statusCode >= 400) { handleResponseError(table, request, qResponse); } @@ -950,9 +957,22 @@ public class BaseAPIActionUtil throw (new QException(rle)); } - LOG.info("Caught RateLimitException", logPair("rateLimitsCaught", rateLimitsCaught), logPair("uri", request.getURI()), logPair("table", table.getName()), logPair("sleeping", sleepMillis)); - SleepUtils.sleep(sleepMillis, TimeUnit.MILLISECONDS); - sleepMillis *= 2; + LOG.info("Caught RateLimitException", logPair("rateLimitsCaught", rateLimitsCaught), logPair("uri", request.getURI()), logPair("table", table.getName()), logPair("sleeping", rateLimitSleepMillis)); + SleepUtils.sleep(rateLimitSleepMillis, TimeUnit.MILLISECONDS); + rateLimitSleepMillis *= 2; + } + catch(RetryableServerErrorException see) + { + serverErrorsCaught++; + if(serverErrorsCaught > getMaxAllowedServerErrors()) + { + LOG.error("Giving up " + request.getMethod() + " to [" + table.getName() + "] after too many server-side errors (" + getMaxAllowedServerErrors() + ")"); + throw (new QException(see)); + } + + LOG.info("Caught Server-side error during API request", logPair("serverErrorsCaught", serverErrorsCaught), logPair("uri", request.getURI()), logPair("table", table.getName()), logPair("sleeping", serverErrorsSleepMillis)); + SleepUtils.sleep(serverErrorsSleepMillis, TimeUnit.MILLISECONDS); + serverErrorsSleepMillis *= 2; } catch(QException qe) { @@ -972,6 +992,16 @@ public class BaseAPIActionUtil + /******************************************************************************* + ** + *******************************************************************************/ + private boolean shouldBeRetryableServerErrorException(QHttpResponse qResponse) + { + return (qResponse.getStatusCode() != null && qResponse.getStatusCode() >= 500); + } + + + /******************************************************************************* ** one-line method, factored out so mock/tests can override *******************************************************************************/ @@ -1141,6 +1171,16 @@ public class BaseAPIActionUtil + /******************************************************************************* + ** + *******************************************************************************/ + protected int getInitialServerErrorBackoffMillis() + { + return (500); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -1151,6 +1191,16 @@ public class BaseAPIActionUtil + /******************************************************************************* + ** + *******************************************************************************/ + protected int getMaxAllowedServerErrors() + { + return (3); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/RetryableServerErrorException.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/RetryableServerErrorException.java new file mode 100644 index 00000000..33f79016 --- /dev/null +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/RetryableServerErrorException.java @@ -0,0 +1,42 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.api.exceptions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class RetryableServerErrorException extends QException +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public RetryableServerErrorException(String message) + { + super(message); + } + +} diff --git a/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtilTest.java b/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtilTest.java index 3930d5b6..a2f195b1 100644 --- a/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtilTest.java +++ b/qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtilTest.java @@ -124,9 +124,16 @@ class BaseAPIActionUtilTest extends BaseTest // avoid the fully mocked makeRequest // //////////////////////////////////////// mockApiUtilsHelper.setUseMock(false); - mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent(""" - {"error": "Server error"} - """)); + + ////////////////////////// + // set to retry 3 times // + ////////////////////////// + for(int i = 0; i < 4; i++) + { + mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent(""" + {"error": "Server error"} + """)); + } CountInput countInput = new CountInput(); countInput.setTableName(TestUtils.MOCK_TABLE_NAME); @@ -290,9 +297,16 @@ class BaseAPIActionUtilTest extends BaseTest // avoid the fully mocked makeRequest // //////////////////////////////////////// mockApiUtilsHelper.setUseMock(false); - mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent(""" - {"error": "Server error"} - """)); + + ////////////////////////// + // set to retry 3 times // + ////////////////////////// + for(int i = 0; i < 4; i++) + { + mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent(""" + {"error": "Server error"} + """)); + } QueryInput queryInput = new QueryInput(); queryInput.setTableName(TestUtils.MOCK_TABLE_NAME); @@ -344,9 +358,16 @@ class BaseAPIActionUtilTest extends BaseTest // avoid the fully mocked makeRequest // //////////////////////////////////////// mockApiUtilsHelper.setUseMock(false); - mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent(""" - {"error": "Server error"} - """)); + + ////////////////////////// + // set to retry 3 times // + ////////////////////////// + for(int i = 0; i < 4; i++) + { + mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent(""" + {"error": "Server error"} + """)); + } InsertInput insertInput = new InsertInput(); insertInput.setRecords(List.of(new QRecord().withValue("name", "Milhouse"))); @@ -411,9 +432,13 @@ class BaseAPIActionUtilTest extends BaseTest // avoid the fully mocked makeRequest // //////////////////////////////////////// mockApiUtilsHelper.setUseMock(false); - mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent(""" - {"error": "Server error"} - """)); + + for(int i = 0; i < 4; i++) + { + mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent(""" + {"error": "Server error"} + """)); + } UpdateInput updateInput = new UpdateInput(); updateInput.setRecords(List.of(new QRecord().withValue("name", "Milhouse"))); @@ -682,4 +707,20 @@ class BaseAPIActionUtilTest extends BaseTest return (new GetAction().execute(getInput)); } -} \ No newline at end of file + + + /******************************************************************************* + ** subclass of base api action utils that can be used to test overriding methods + *******************************************************************************/ + private class BaseAPIActionUtilSubclass extends BaseAPIActionUtil + { + /******************************************************************************* + ** + *******************************************************************************/ + private boolean shouldBeRetryableServerErrorException(QHttpResponse qResponse) + { + return (false); + } + + } +}