QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/sprint-14' into feature/sprint-15

Browse Source
This commit is contained in:
Tim Chamberlain
2022-11-03 14:36:32 -05:00
parent 99561ecf81 2105e3dfc5
commit ebf9556a5d
3 changed files with 70 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIGetAction.java
View File

@ -29,8 +29,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -53,13 +53,11 @@ public class APIGetAction extends AbstractAPIAction implements GetInterface
QTableMetaData table = getInput.getTable();
preAction(getInput);
try
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
try(CloseableHttpClient client = httpClientBuilder.build())
{
String urlSuffix = apiActionUtil.buildUrlSuffixForSingleRecordGet(getInput.getPrimaryKey());
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
HttpClient client = httpClientBuilder.build();
String url = apiActionUtil.buildTableUrl(table);
HttpGet request = new HttpGet(url + urlSuffix);

52
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIUpdateAction.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.module.api.actions;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -34,7 +35,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
import com.kingsrook.qqq.backend.module.api.exceptions.RateLimitException;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
@ -76,11 +77,12 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
{
connectionManager = new PoolingHttpClientConnectionManager();
// todo - supports bulk put?
for(QRecord record : updateInput.getRecords())
///////////////////////////////////////////////////////////////
// make post requests for groups of orders that need updated //
///////////////////////////////////////////////////////////////
for(List<QRecord> recordList : CollectionUtils.getPages(updateInput.getRecords(), 20))
{
putRecords(updateOutput, table, connectionManager, record);
processRecords(table, connectionManager, recordList);
if(updateInput.getRecords().size() > 1 && apiActionUtil.getMillisToSleepAfterEveryCall() > 0)
{
@ -110,7 +112,7 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
/*******************************************************************************
**
*******************************************************************************/
private void putRecords(UpdateOutput updateOutput, QTableMetaData table, HttpClientConnectionManager connectionManager, QRecord record) throws RateLimitException
private void processRecords(QTableMetaData table, HttpClientConnectionManager connectionManager, List<QRecord> recordList)
{
int sleepMillis = apiActionUtil.getInitialRateLimitBackoffMillis();
int rateLimitsCaught = 0;
@ -118,7 +120,7 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
{
try
{
putOneTime(updateOutput, table, connectionManager, record);
doPost(table, connectionManager, recordList);
return;
}
catch(RateLimitException rle)
@ -127,12 +129,10 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
if(rateLimitsCaught > apiActionUtil.getMaxAllowedRateLimitErrors())
{
LOG.warn("Giving up PUT to [" + table.getName() + "] after too many rate-limit errors (" + apiActionUtil.getMaxAllowedRateLimitErrors() + ")");
record.addError("Error: " + rle.getMessage());
updateOutput.addRecord(record);
return;
}
LOG.info("Caught RateLimitException [#" + rateLimitsCaught + "] PUT'ing to [" + table.getName() + "] - sleeping [" + sleepMillis + "]...");
LOG.info("Caught RateLimitException [#" + rateLimitsCaught + "] POSTing to [" + table.getName() + "] - sleeping [" + sleepMillis + "]...");
SleepUtils.sleep(sleepMillis, TimeUnit.MILLISECONDS);
sleepMillis *= 2;
}
@ -144,20 +144,17 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
/*******************************************************************************
**
*******************************************************************************/
private void putOneTime(UpdateOutput insertOutput, QTableMetaData table, HttpClientConnectionManager connectionManager, QRecord record) throws RateLimitException
private void doPost(QTableMetaData table, HttpClientConnectionManager connectionManager, List<QRecord> recordList) throws RateLimitException
{
try
try(CloseableHttpClient client = HttpClients.custom().setConnectionManager(connectionManager).build())
{
CloseableHttpClient client = HttpClients.custom().setConnectionManager(connectionManager).build();
String url = buildTableUrl(table);
url += record.getValueString("number");
HttpPut request = new HttpPut(url);
String url = apiActionUtil.buildTableUrl(table);
HttpPost request = new HttpPost(url);
apiActionUtil.setupAuthorizationInRequest(request);
apiActionUtil.setupContentTypeInRequest(request);
apiActionUtil.setupAdditionalHeaders(request);
request.setEntity(apiActionUtil.recordToEntity(table, record));
request.setEntity(apiActionUtil.recordsToEntity(table, recordList));
HttpResponse response = client.execute(request);
int statusCode = response.getStatusLine().getStatusCode();
@ -165,9 +162,10 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
{
throw (new RateLimitException(EntityUtils.toString(response.getEntity())));
}
QRecord outputRecord = apiActionUtil.processPostResponse(table, record, response);
insertOutput.addRecord(outputRecord);
if(statusCode != 207)
{
LOG.warn("Did not receive response status code of 207: " + EntityUtils.toString(response.getEntity()));
}
}
catch(RateLimitException rle)
{
@ -176,19 +174,7 @@ public class APIUpdateAction extends AbstractAPIAction implements UpdateInterfac
catch(Exception e)
{
LOG.warn("Error posting to [" + table.getName() + "]", e);
record.addError("Error: " + e.getMessage());
insertOutput.addRecord(record);
}
}
/*******************************************************************************
**
*******************************************************************************/
protected String buildTableUrl(QTableMetaData table)
{
return (backendMetaData.getBaseUrl() + "/orders/SalesOrder/");
}
}

48
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java
View File

@ -30,6 +30,7 @@ import java.util.Base64;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@ -245,6 +246,34 @@ public class BaseAPIActionUtil
/*******************************************************************************
** Build an HTTP Entity (e.g., for a PUT or POST) from a list of QRecords. Can be
** overridden if an API doesn't do a basic json object. Or, can override a
** helper method, such as recordToJsonObject.
**
*******************************************************************************/
protected AbstractHttpEntity recordsToEntity(QTableMetaData table, List<QRecord> recordList) throws IOException
{
JSONArray entityListJson = new JSONArray();
for(QRecord record : recordList)
{
entityListJson.put(entityListJson.length(), recordToJsonObject(table, record));
}
String json = entityListJson.toString();
String tablePath = getBackendDetails(table).getTablePath();
if(tablePath != null)
{
JSONObject body = new JSONObject();
body.put(tablePath, new JSONArray(json));
json = body.toString();
}
LOG.debug(json);
return (new StringEntity(json));
}
/*******************************************************************************
** Helper for recordToEntity - builds a basic JSON object. Can be
** overridden if an API doesn't do a basic json object.
@ -515,4 +544,23 @@ public class BaseAPIActionUtil
return (records == null ? null : records.size());
}
/*******************************************************************************
**
*******************************************************************************/
protected void throwUnsupportedCriteriaField(QFilterCriteria criteria) throws QUserFacingException
{
throw new QUserFacingException("Unsupported query field [" + criteria.getFieldName() + "]");
}
/*******************************************************************************
**
*******************************************************************************/
protected void throwUnsupportedCriteriaOperator(QFilterCriteria criteria) throws QUserFacingException
{
throw new QUserFacingException("Unsupported operator [" + criteria.getOperator() + "] for query field [" + criteria.getFieldName() + "]");
}
}