updated to allow 'trying again' when server side 500 error occur in makeRequest()

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -63,6 +63,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.module.api.exceptions.OAuthCredentialsException;
 import com.kingsrook.qqq.backend.module.api.exceptions.OAuthExpiredTokenException;
 import com.kingsrook.qqq.backend.module.api.exceptions.RateLimitException;
+import com.kingsrook.qqq.backend.module.api.exceptions.RetryableServerErrorException;
 import com.kingsrook.qqq.backend.module.api.model.AuthorizationType;
 import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog;
 import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendMetaData;
@@ -878,8 +879,10 @@ public class BaseAPIActionUtil
     *******************************************************************************/
    public QHttpResponse makeRequest(QTableMetaData table, HttpRequestBase request) throws QException
    {
-      int     sleepMillis               = getInitialRateLimitBackoffMillis();
+      int     rateLimitSleepMillis      = getInitialRateLimitBackoffMillis();
+      int     serverErrorsSleepMillis   = getInitialServerErrorBackoffMillis();
       int     rateLimitsCaught          = 0;
+      int     serverErrorsCaught        = 0;
       boolean caughtAnOAuthExpiredToken = false;
 
       while(true)
@@ -913,7 +916,11 @@ public class BaseAPIActionUtil
                {
                   throw (new RateLimitException(qResponse.getContent()));
                }
-               if(statusCode >= 400)
+               else if(shouldBeRetryableServerErrorException(qResponse))
+               {
+                  throw (new RetryableServerErrorException(qResponse.getContent()));
+               }
+               else if(statusCode >= 400)
                {
                   handleResponseError(table, request, qResponse);
                }
@@ -950,9 +957,22 @@ public class BaseAPIActionUtil
                throw (new QException(rle));
             }
 
-            LOG.info("Caught RateLimitException", logPair("rateLimitsCaught", rateLimitsCaught), logPair("uri", request.getURI()), logPair("table", table.getName()), logPair("sleeping", sleepMillis));
+            LOG.info("Caught RateLimitException", logPair("rateLimitsCaught", rateLimitsCaught), logPair("uri", request.getURI()), logPair("table", table.getName()), logPair("sleeping", rateLimitSleepMillis));
-            SleepUtils.sleep(sleepMillis, TimeUnit.MILLISECONDS);
+            SleepUtils.sleep(rateLimitSleepMillis, TimeUnit.MILLISECONDS);
-            sleepMillis *= 2;
+            rateLimitSleepMillis *= 2;
+         }
+         catch(RetryableServerErrorException see)
+         {
+            serverErrorsCaught++;
+            if(serverErrorsCaught > getMaxAllowedServerErrors())
+            {
+               LOG.error("Giving up " + request.getMethod() + " to [" + table.getName() + "] after too many server-side errors (" + getMaxAllowedServerErrors() + ")");
+               throw (new QException(see));
+            }
+
+            LOG.info("Caught Server-side error during API request", logPair("serverErrorsCaught", serverErrorsCaught), logPair("uri", request.getURI()), logPair("table", table.getName()), logPair("sleeping", serverErrorsSleepMillis));
+            SleepUtils.sleep(serverErrorsSleepMillis, TimeUnit.MILLISECONDS);
+            serverErrorsSleepMillis *= 2;
          }
          catch(QException qe)
          {
@@ -972,6 +992,16 @@ public class BaseAPIActionUtil
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private boolean shouldBeRetryableServerErrorException(QHttpResponse qResponse)
+   {
+      return (qResponse.getStatusCode() != null && qResponse.getStatusCode() >= 500);
+   }
+
+
+
    /*******************************************************************************
     ** one-line method, factored out so mock/tests can override
     *******************************************************************************/
@@ -1141,6 +1171,16 @@ public class BaseAPIActionUtil
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected int getInitialServerErrorBackoffMillis()
+   {
+      return (500);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -1151,6 +1191,16 @@ public class BaseAPIActionUtil
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected int getMaxAllowedServerErrors()
+   {
+      return (3);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/exceptions/RetryableServerErrorException.java

@@ -0,0 +1,42 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.exceptions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class RetryableServerErrorException extends QException
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public RetryableServerErrorException(String message)
+   {
+      super(message);
+   }
+
+}

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtilTest.java

@@ -124,9 +124,16 @@ class BaseAPIActionUtilTest extends BaseTest
       // avoid the fully mocked makeRequest //
       ////////////////////////////////////////
       mockApiUtilsHelper.setUseMock(false);
-      mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
+
-         {"error": "Server error"}
+      //////////////////////////
-         """));
+      // set to retry 3 times //
+      //////////////////////////
+      for(int i = 0; i < 4; i++)
+      {
+         mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
+            {"error": "Server error"}
+            """));
+      }
 
       CountInput countInput = new CountInput();
       countInput.setTableName(TestUtils.MOCK_TABLE_NAME);
@@ -290,9 +297,16 @@ class BaseAPIActionUtilTest extends BaseTest
       // avoid the fully mocked makeRequest //
       ////////////////////////////////////////
       mockApiUtilsHelper.setUseMock(false);
-      mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
+
-         {"error": "Server error"}
+      //////////////////////////
-         """));
+      // set to retry 3 times //
+      //////////////////////////
+      for(int i = 0; i < 4; i++)
+      {
+         mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
+            {"error": "Server error"}
+            """));
+      }
 
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(TestUtils.MOCK_TABLE_NAME);
@@ -344,9 +358,16 @@ class BaseAPIActionUtilTest extends BaseTest
       // avoid the fully mocked makeRequest //
       ////////////////////////////////////////
       mockApiUtilsHelper.setUseMock(false);
-      mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
+
-         {"error": "Server error"}
+      //////////////////////////
-         """));
+      // set to retry 3 times //
+      //////////////////////////
+      for(int i = 0; i < 4; i++)
+      {
+         mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
+            {"error": "Server error"}
+            """));
+      }
 
       InsertInput insertInput = new InsertInput();
       insertInput.setRecords(List.of(new QRecord().withValue("name", "Milhouse")));
@@ -411,9 +432,13 @@ class BaseAPIActionUtilTest extends BaseTest
       // avoid the fully mocked makeRequest //
       ////////////////////////////////////////
       mockApiUtilsHelper.setUseMock(false);
-      mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
+
-         {"error": "Server error"}
+      for(int i = 0; i < 4; i++)
-         """));
+      {
+         mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(500).withContent("""
+            {"error": "Server error"}
+            """));
+      }
 
       UpdateInput updateInput = new UpdateInput();
       updateInput.setRecords(List.of(new QRecord().withValue("name", "Milhouse")));
@@ -682,4 +707,20 @@ class BaseAPIActionUtilTest extends BaseTest
       return (new GetAction().execute(getInput));
    }
 
+
+
+   /*******************************************************************************
+    ** subclass of base api action utils that can be used to test overriding methods
+    *******************************************************************************/
+   private class BaseAPIActionUtilSubclass extends BaseAPIActionUtil
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      private boolean shouldBeRetryableServerErrorException(QHttpResponse qResponse)
+      {
+         return (false);
+      }
+
+   }
 }