QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

SPRINT-15: updated apiupdateAction to handle 200/207 better, added better exception handling

Browse Source
This commit is contained in:
Tim Chamberlain
2022-11-08 16:33:06 -06:00
parent 8932ef891e
commit 1e09931218
3 changed files with 35 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIInsertAction.java
View File

@ -153,10 +153,8 @@ public class APIInsertAction extends AbstractAPIAction implements InsertInterfac
*******************************************************************************/
private void postOneTime(InsertOutput insertOutput, QTableMetaData table, HttpClientConnectionManager connectionManager, QRecord record) throws RateLimitException
{
try
try(CloseableHttpClient client = HttpClients.custom().setConnectionManager(connectionManager).build())
{
CloseableHttpClient client = HttpClients.custom().setConnectionManager(connectionManager).build();
String url = apiActionUtil.buildTableUrl(table);
HttpPost request = new HttpPost(url);
apiActionUtil.setupAuthorizationInRequest(request);
@ -186,5 +184,4 @@ public class APIInsertAction extends AbstractAPIAction implements InsertInterfac
insertOutput.addRecord(record);
}
}
}

46
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIUpdateAction.java
View File

@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import com.kingsrook.qqq.backend.module.api.exceptions.RateLimitException;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.impl.client.CloseableHttpClient;
@ -43,6 +44,7 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONObject;
/*******************************************************************************
@ -83,8 +85,11 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
for(List<QRecord> recordList : CollectionUtils.getPages(updateInput.getRecords(), 20))
{
processRecords(table, connectionManager, recordList);
if(updateInput.getRecords().size() > 1 && apiActionUtil.getMillisToSleepAfterEveryCall() > 0)
for(QRecord qRecord : recordList)
{
updateOutput.addRecord(qRecord);
}
if(recordList.size() == 20 && apiActionUtil.getMillisToSleepAfterEveryCall() > 0)
{
SleepUtils.sleep(apiActionUtil.getMillisToSleepAfterEveryCall(), TimeUnit.MILLISECONDS);
}
@ -112,7 +117,7 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
/*******************************************************************************
**
*******************************************************************************/
private void processRecords(QTableMetaData table, HttpClientConnectionManager connectionManager, List<QRecord> recordList)
private void processRecords(QTableMetaData table, HttpClientConnectionManager connectionManager, List<QRecord> recordList) throws QException
{
int sleepMillis = apiActionUtil.getInitialRateLimitBackoffMillis();
int rateLimitsCaught = 0;
@ -144,7 +149,7 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
/*******************************************************************************
**
*******************************************************************************/
private void doPost(QTableMetaData table, HttpClientConnectionManager connectionManager, List<QRecord> recordList) throws RateLimitException
private void doPost(QTableMetaData table, HttpClientConnectionManager connectionManager, List<QRecord> recordList) throws RateLimitException, QException
{
try(CloseableHttpClient client = HttpClients.custom().setConnectionManager(connectionManager).build())
{
@ -156,24 +161,39 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
request.setEntity(apiActionUtil.recordsToEntity(table, recordList));
HttpResponse response = client.execute(request);
int statusCode = response.getStatusLine().getStatusCode();
if(statusCode == 429)
HttpResponse response = client.execute(request);
int statusCode = response.getStatusLine().getStatusCode();
String responseString = EntityUtils.toString(response.getEntity());
if(statusCode == HttpStatus.SC_TOO_MANY_REQUESTS)
{
throw (new RateLimitException(EntityUtils.toString(response.getEntity())));
throw (new RateLimitException(responseString));
}
if(statusCode != 207)
if(statusCode != HttpStatus.SC_MULTI_STATUS && statusCode != HttpStatus.SC_OK)
{
LOG.warn("Did not receive response status code of 207: " + EntityUtils.toString(response.getEntity()));
String errorMessage = "Did not receive response status code of 200 or 207: " + responseString;
LOG.warn(errorMessage);
throw (new QException(errorMessage));
}
if(statusCode == HttpStatus.SC_MULTI_STATUS)
{
JSONObject responseJSON = new JSONObject(responseString).getJSONObject("response");
if(!responseJSON.optString("status").contains("200 OK"))
{
String errorMessage = "Did not receive ok status response: " + responseJSON.optString("description");
LOG.warn(errorMessage);
throw (new QException(errorMessage));
}
}
}
catch(RateLimitException rle)
catch(RateLimitException | QException e)
{
throw (rle);
throw (e);
}
catch(Exception e)
{
LOG.warn("Error posting to [" + table.getName() + "]", e);
String errorMessage = "An unexpected error occurred updating entities.";
LOG.warn(errorMessage, e);
throw (new QException(errorMessage, e));
}
}

2
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java
View File

@ -252,7 +252,7 @@ public class BaseAPIActionUtil
** helper method, such as recordToJsonObject.
**
*******************************************************************************/
protected AbstractHttpEntity recordsToEntity(QTableMetaData table, List<QRecord> recordList) throws IOException
protected AbstractHttpEntity recordsToEntity(QTableMetaData table, List<QRecord> recordList) throws IOException, QException
{
JSONArray entityListJson = new JSONArray();
for(QRecord record : recordList)