From 1e09931218c8ff28b6f325e19b86c70bf0a924fe Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 8 Nov 2022 16:33:06 -0600 Subject: [PATCH] SPRINT-15: updated apiupdateAction to handle 200/207 better, added better exception handling --- .../module/api/actions/APIInsertAction.java | 5 +- .../module/api/actions/APIUpdateAction.java | 46 +++++++++++++------ .../module/api/actions/BaseAPIActionUtil.java | 2 +- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIInsertAction.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIInsertAction.java index 375450cd..9b69a342 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIInsertAction.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIInsertAction.java @@ -153,10 +153,8 @@ public class APIInsertAction extends AbstractAPIAction implements InsertInterfac *******************************************************************************/ private void postOneTime(InsertOutput insertOutput, QTableMetaData table, HttpClientConnectionManager connectionManager, QRecord record) throws RateLimitException { - try + try(CloseableHttpClient client = HttpClients.custom().setConnectionManager(connectionManager).build()) { - CloseableHttpClient client = HttpClients.custom().setConnectionManager(connectionManager).build(); - String url = apiActionUtil.buildTableUrl(table); HttpPost request = new HttpPost(url); apiActionUtil.setupAuthorizationInRequest(request); @@ -186,5 +184,4 @@ public class APIInsertAction extends AbstractAPIAction implements InsertInterfac insertOutput.addRecord(record); } } - } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIUpdateAction.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIUpdateAction.java index bfa36b1b..5593c612 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIUpdateAction.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIUpdateAction.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.module.api.exceptions.RateLimitException; import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; import org.apache.http.client.methods.HttpPost; import org.apache.http.conn.HttpClientConnectionManager; import org.apache.http.impl.client.CloseableHttpClient; @@ -43,6 +44,7 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.json.JSONObject; /******************************************************************************* @@ -83,8 +85,11 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac for(List recordList : CollectionUtils.getPages(updateInput.getRecords(), 20)) { processRecords(table, connectionManager, recordList); - - if(updateInput.getRecords().size() > 1 && apiActionUtil.getMillisToSleepAfterEveryCall() > 0) + for(QRecord qRecord : recordList) + { + updateOutput.addRecord(qRecord); + } + if(recordList.size() == 20 && apiActionUtil.getMillisToSleepAfterEveryCall() > 0) { SleepUtils.sleep(apiActionUtil.getMillisToSleepAfterEveryCall(), TimeUnit.MILLISECONDS); } @@ -112,7 +117,7 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac /******************************************************************************* ** *******************************************************************************/ - private void processRecords(QTableMetaData table, HttpClientConnectionManager connectionManager, List recordList) + private void processRecords(QTableMetaData table, HttpClientConnectionManager connectionManager, List recordList) throws QException { int sleepMillis = apiActionUtil.getInitialRateLimitBackoffMillis(); int rateLimitsCaught = 0; @@ -144,7 +149,7 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac /******************************************************************************* ** *******************************************************************************/ - private void doPost(QTableMetaData table, HttpClientConnectionManager connectionManager, List recordList) throws RateLimitException + private void doPost(QTableMetaData table, HttpClientConnectionManager connectionManager, List recordList) throws RateLimitException, QException { try(CloseableHttpClient client = HttpClients.custom().setConnectionManager(connectionManager).build()) { @@ -156,24 +161,39 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac request.setEntity(apiActionUtil.recordsToEntity(table, recordList)); - HttpResponse response = client.execute(request); - int statusCode = response.getStatusLine().getStatusCode(); - if(statusCode == 429) + HttpResponse response = client.execute(request); + int statusCode = response.getStatusLine().getStatusCode(); + String responseString = EntityUtils.toString(response.getEntity()); + if(statusCode == HttpStatus.SC_TOO_MANY_REQUESTS) { - throw (new RateLimitException(EntityUtils.toString(response.getEntity()))); + throw (new RateLimitException(responseString)); } - if(statusCode != 207) + if(statusCode != HttpStatus.SC_MULTI_STATUS && statusCode != HttpStatus.SC_OK) { - LOG.warn("Did not receive response status code of 207: " + EntityUtils.toString(response.getEntity())); + String errorMessage = "Did not receive response status code of 200 or 207: " + responseString; + LOG.warn(errorMessage); + throw (new QException(errorMessage)); + } + if(statusCode == HttpStatus.SC_MULTI_STATUS) + { + JSONObject responseJSON = new JSONObject(responseString).getJSONObject("response"); + if(!responseJSON.optString("status").contains("200 OK")) + { + String errorMessage = "Did not receive ok status response: " + responseJSON.optString("description"); + LOG.warn(errorMessage); + throw (new QException(errorMessage)); + } } } - catch(RateLimitException rle) + catch(RateLimitException | QException e) { - throw (rle); + throw (e); } catch(Exception e) { - LOG.warn("Error posting to [" + table.getName() + "]", e); + String errorMessage = "An unexpected error occurred updating entities."; + LOG.warn(errorMessage, e); + throw (new QException(errorMessage, e)); } } diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 8034f5a0..ee87b2a6 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -252,7 +252,7 @@ public class BaseAPIActionUtil ** helper method, such as recordToJsonObject. ** *******************************************************************************/ - protected AbstractHttpEntity recordsToEntity(QTableMetaData table, List recordList) throws IOException + protected AbstractHttpEntity recordsToEntity(QTableMetaData table, List recordList) throws IOException, QException { JSONArray entityListJson = new JSONArray(); for(QRecord record : recordList)