QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

SPRINT-15: updated apiupdateAction to handle 200/207 better, added better exception handling

Browse Source
This commit is contained in:
Tim Chamberlain
2022-11-08 16:33:06 -06:00
parent 8932ef891e
commit 1e09931218
3 changed files with 35 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIInsertAction.java
View File

@ -153,10 +153,8 @@ public class APIInsertAction extends AbstractAPIAction implements InsertInterfac
*******************************************************************************/ *******************************************************************************/
private void postOneTime(InsertOutput insertOutput, QTableMetaData table, HttpClientConnectionManager connectionManager, QRecord record) throws RateLimitException private void postOneTime(InsertOutput insertOutput, QTableMetaData table, HttpClientConnectionManager connectionManager, QRecord record) throws RateLimitException
{ {
try try(CloseableHttpClient client = HttpClients.custom().setConnectionManager(connectionManager).build())
{ {
CloseableHttpClient client = HttpClients.custom().setConnectionManager(connectionManager).build();
String url = apiActionUtil.buildTableUrl(table); String url = apiActionUtil.buildTableUrl(table);
HttpPost request = new HttpPost(url); HttpPost request = new HttpPost(url);
apiActionUtil.setupAuthorizationInRequest(request); apiActionUtil.setupAuthorizationInRequest(request);
@ -186,5 +184,4 @@ public class APIInsertAction extends AbstractAPIAction implements InsertInterfac
insertOutput.addRecord(record); insertOutput.addRecord(record);
} }
} }
} }

46
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIUpdateAction.java
View File

@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import com.kingsrook.qqq.backend.module.api.exceptions.RateLimitException; import com.kingsrook.qqq.backend.module.api.exceptions.RateLimitException;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPost;
import org.apache.http.conn.HttpClientConnectionManager; import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
@ -43,6 +44,7 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.json.JSONObject;
/******************************************************************************* /*******************************************************************************
@ -83,8 +85,11 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
for(List<QRecord> recordList : CollectionUtils.getPages(updateInput.getRecords(), 20)) for(List<QRecord> recordList : CollectionUtils.getPages(updateInput.getRecords(), 20))
{ {
processRecords(table, connectionManager, recordList); processRecords(table, connectionManager, recordList);
for(QRecord qRecord : recordList)
if(updateInput.getRecords().size() > 1 && apiActionUtil.getMillisToSleepAfterEveryCall() > 0) {
updateOutput.addRecord(qRecord);
}
if(recordList.size() == 20 && apiActionUtil.getMillisToSleepAfterEveryCall() > 0)
{ {
SleepUtils.sleep(apiActionUtil.getMillisToSleepAfterEveryCall(), TimeUnit.MILLISECONDS); SleepUtils.sleep(apiActionUtil.getMillisToSleepAfterEveryCall(), TimeUnit.MILLISECONDS);
} }
@ -112,7 +117,7 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/
private void processRecords(QTableMetaData table, HttpClientConnectionManager connectionManager, List<QRecord> recordList) private void processRecords(QTableMetaData table, HttpClientConnectionManager connectionManager, List<QRecord> recordList) throws QException
{ {
int sleepMillis = apiActionUtil.getInitialRateLimitBackoffMillis(); int sleepMillis = apiActionUtil.getInitialRateLimitBackoffMillis();
int rateLimitsCaught = 0; int rateLimitsCaught = 0;
@ -144,7 +149,7 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/
private void doPost(QTableMetaData table, HttpClientConnectionManager connectionManager, List<QRecord> recordList) throws RateLimitException private void doPost(QTableMetaData table, HttpClientConnectionManager connectionManager, List<QRecord> recordList) throws RateLimitException, QException
{ {
try(CloseableHttpClient client = HttpClients.custom().setConnectionManager(connectionManager).build()) try(CloseableHttpClient client = HttpClients.custom().setConnectionManager(connectionManager).build())
{ {
@ -156,24 +161,39 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
request.setEntity(apiActionUtil.recordsToEntity(table, recordList)); request.setEntity(apiActionUtil.recordsToEntity(table, recordList));
HttpResponse response = client.execute(request); HttpResponse response = client.execute(request);
int statusCode = response.getStatusLine().getStatusCode(); int statusCode = response.getStatusLine().getStatusCode();
if(statusCode == 429) String responseString = EntityUtils.toString(response.getEntity());
if(statusCode == HttpStatus.SC_TOO_MANY_REQUESTS)
{ {
throw (new RateLimitException(EntityUtils.toString(response.getEntity()))); throw (new RateLimitException(responseString));
} }
if(statusCode != 207) if(statusCode != HttpStatus.SC_MULTI_STATUS && statusCode != HttpStatus.SC_OK)
{ {
LOG.warn("Did not receive response status code of 207: " + EntityUtils.toString(response.getEntity())); String errorMessage = "Did not receive response status code of 200 or 207: " + responseString;
LOG.warn(errorMessage);
throw (new QException(errorMessage));
}
if(statusCode == HttpStatus.SC_MULTI_STATUS)
{
JSONObject responseJSON = new JSONObject(responseString).getJSONObject("response");
if(!responseJSON.optString("status").contains("200 OK"))
{
String errorMessage = "Did not receive ok status response: " + responseJSON.optString("description");
LOG.warn(errorMessage);
throw (new QException(errorMessage));
}
} }
} }
catch(RateLimitException rle) catch(RateLimitException | QException e)
{ {
throw (rle); throw (e);
} }
catch(Exception e) catch(Exception e)
{ {
LOG.warn("Error posting to [" + table.getName() + "]", e); String errorMessage = "An unexpected error occurred updating entities.";
LOG.warn(errorMessage, e);
throw (new QException(errorMessage, e));
} }
} }

2
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java
View File

@ -252,7 +252,7 @@ public class BaseAPIActionUtil
** helper method, such as recordToJsonObject. ** helper method, such as recordToJsonObject.
** **
*******************************************************************************/ *******************************************************************************/
protected AbstractHttpEntity recordsToEntity(QTableMetaData table, List<QRecord> recordList) throws IOException protected AbstractHttpEntity recordsToEntity(QTableMetaData table, List<QRecord> recordList) throws IOException, QException
{ {
JSONArray entityListJson = new JSONArray(); JSONArray entityListJson = new JSONArray();
for(QRecord record : recordList) for(QRecord record : recordList)