QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ExtractViaBasepullQueryStep; add pagination & piping to api query

Browse Source
This commit is contained in:
darin.kelkhoff
2022-10-26 12:24:35 -05:00
parent 77927fd318
commit 82810c2b66
18 changed files with 689 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java
View File

@ -52,7 +52,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
import com.kingsrook.qqq.backend.core.state.StateType;
@ -69,9 +69,11 @@ import org.apache.logging.log4j.Logger;
*******************************************************************************/
public class RunProcessAction
{
private static final Logger LOG = LogManager.getLogger(RunProcessAction.class);
public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
private static final Logger LOG = LogManager.getLogger(RunProcessAction.class);
public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
public static final String BASEPULL_TIMESTAMP_FIELD = "basepullTimestampField";
@ -422,7 +424,7 @@ public class RunProcessAction
/*******************************************************************************
**
** Insert or update the last runtime value for this basepull into the backend.
*******************************************************************************/
protected void storeLastRunTime(RunProcessInput runProcessInput, QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
{
@ -489,13 +491,22 @@ public class RunProcessAction
/*******************************************************************************
**
** Lookup the last runtime for this basepull, and set it (plus now) in the process's
** values.
*******************************************************************************/
protected void persistLastRunTime(RunProcessInput runProcessInput, QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
{
////////////////////////////////////////////////////////////////////////////////////////////////
// store 'now', which will be used to update basepull record if process completes sucessfully //
////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////
// if these values were already computed, don't re-do //
////////////////////////////////////////////////////////
if(runProcessInput.getValue(BASEPULL_THIS_RUNTIME_KEY) != null)
{
return;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// store 'now', which will be used to update basepull record if process completes successfully //
/////////////////////////////////////////////////////////////////////////////////////////////////
Instant now = Instant.now();
runProcessInput.getValues().put(BASEPULL_THIS_RUNTIME_KEY, now);
@ -533,5 +544,6 @@ public class RunProcessAction
}
runProcessInput.getValues().put(BASEPULL_LAST_RUNTIME_KEY, lastRunTime);
runProcessInput.getValues().put(BASEPULL_TIMESTAMP_FIELD, basepullConfiguration.getTimestampField());
}
}

18
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -316,6 +316,24 @@ public class QInstanceEnricher
{
generateAppSections(app);
}
for(QAppSection section : CollectionUtils.nonNullList(app.getSections()))
{
enrichAppSection(section);
}
}
/*******************************************************************************
**
*******************************************************************************/
private void enrichAppSection(QAppSection section)
{
if(!StringUtils.hasContent(section.getLabel()))
{
section.setLabel(nameToLabel(section.getName()));
}
}

11
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java
View File

@ -419,6 +419,17 @@ public class RunBackendStepInput extends AbstractActionInput
/*******************************************************************************
** Getter for a single field's value
**
*******************************************************************************/
public Instant getValueInstant(String fieldName)
{
return (ValueUtils.getValueAsInstant(getValue(fieldName)));
}
/*******************************************************************************
** Accessor for processState - protected, because we generally want to access
** its members through wrapper methods, we think

47
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java
View File

@ -25,6 +25,9 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
@ -33,6 +36,8 @@ import java.util.List;
*******************************************************************************/
public class QFilterCriteria implements Serializable, Cloneable
{
private static final Logger LOG = LogManager.getLogger(QFilterCriteria.class);
private String fieldName;
private QCriteriaOperator operator;
private List<Serializable> values;
@ -183,4 +188,46 @@ public class QFilterCriteria implements Serializable, Cloneable
this.values = values;
return this;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
StringBuilder rs = new StringBuilder(fieldName);
try
{
rs.append(" ").append(operator).append(" ");
if(CollectionUtils.nullSafeHasContents(values))
{
if(values.size() == 1)
{
rs.append(values.get(0));
}
else
{
int index = 0;
for(Serializable value : values)
{
if(index++ > 9)
{
rs.append("and ").append(values.size() - index).append(" more");
break;
}
rs.append(value).append(",");
}
}
}
}
catch(Exception e)
{
LOG.warn("Error in toString", e);
rs.append("Error generating toString...");
}
return (rs.toString());
}
}

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterOrderBy.java
View File

@ -151,4 +151,14 @@ public class QFilterOrderBy implements Serializable, Cloneable
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
return (fieldName + " " + (isAscending ? "ASC" : "DESC"));
}
}

41
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java
View File

@ -26,6 +26,8 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
@ -34,6 +36,8 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
*******************************************************************************/
public class QQueryFilter implements Serializable, Cloneable
{
private static final Logger LOG = LogManager.getLogger(QQueryFilter.class);
private List<QFilterCriteria> criteria = new ArrayList<>();
private List<QFilterOrderBy> orderBys = new ArrayList<>();
@ -301,4 +305,41 @@ public class QQueryFilter implements Serializable, Cloneable
subFilters.add(subFilter);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
StringBuilder rs = new StringBuilder("(");
try
{
for(QFilterCriteria criterion : CollectionUtils.nonNullList(criteria))
{
rs.append(criterion).append(" ").append(getBooleanOperator());
}
for(QQueryFilter subFilter : CollectionUtils.nonNullList(subFilters))
{
rs.append(subFilter);
}
rs.append(")");
rs.append("OrderBy[");
for(QFilterOrderBy orderBy : orderBys)
{
rs.append(orderBy).append(",");
}
rs.append("]");
}
catch(Exception e)
{
LOG.warn("Error in toString", e);
rs.append("Error generating toString...");
}
return (rs.toString());
}
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java
View File

@ -33,7 +33,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
/*******************************************************************************

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java
View File

@ -822,4 +822,5 @@ public class QTableMetaData implements QAppChildMetaData, Serializable
return (this);
}
}

247
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/BasepullConfiguration.java Normal file
View File

@ -0,0 +1,247 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
import java.io.Serializable;
/*******************************************************************************
** Class for storing all basepull configuration data
**
*******************************************************************************/
public class BasepullConfiguration implements Serializable
{
private String tableName; // the table that stores the basepull timestamps
private String keyField; // the field in the basepull timestamps table that stores the key of the basepull (e.g., a process name)
private String keyValue; // the key applied to the keyField - optional - if not set, process.getName is used.
private String lastRunTimeFieldName; // the field in the basepull timestamps table that stores the last-run time for the job.
private Integer hoursBackForInitialTimestamp; // for the first-run use-case (where there is no row in the timestamps table), how many hours back in time to look.
private String timestampField; // the name of the field in the table being queried against the last-run timestamp.
/*******************************************************************************
** Getter for tableName
**
*******************************************************************************/
public String getTableName()
{
return tableName;
}
/*******************************************************************************
** Setter for tableName
**
*******************************************************************************/
public void setTableName(String tableName)
{
this.tableName = tableName;
}
/*******************************************************************************
** Fluent setter for tableName
**
*******************************************************************************/
public BasepullConfiguration withTableName(String tableName)
{
this.tableName = tableName;
return (this);
}
/*******************************************************************************
** Getter for keyField
**
*******************************************************************************/
public String getKeyField()
{
return keyField;
}
/*******************************************************************************
** Setter for keyField
**
*******************************************************************************/
public void setKeyField(String keyField)
{
this.keyField = keyField;
}
/*******************************************************************************
** Fluent setter for keyField
**
*******************************************************************************/
public BasepullConfiguration withKeyField(String keyField)
{
this.keyField = keyField;
return (this);
}
/*******************************************************************************
** Getter for keyValue
**
*******************************************************************************/
public String getKeyValue()
{
return keyValue;
}
/*******************************************************************************
** Setter for keyValue
**
*******************************************************************************/
public void setKeyValue(String keyValue)
{
this.keyValue = keyValue;
}
/*******************************************************************************
** Fluent setter for keyValue
**
*******************************************************************************/
public BasepullConfiguration withKeyValue(String keyValue)
{
this.keyValue = keyValue;
return (this);
}
/*******************************************************************************
** Getter for lastRunTimeFieldName
**
*******************************************************************************/
public String getLastRunTimeFieldName()
{
return lastRunTimeFieldName;
}
/*******************************************************************************
** Setter for lastRunTimeFieldName
**
*******************************************************************************/
public void setLastRunTimeFieldName(String lastRunTimeFieldName)
{
this.lastRunTimeFieldName = lastRunTimeFieldName;
}
/*******************************************************************************
** Fluent setter for lastRunTimeFieldName
**
*******************************************************************************/
public BasepullConfiguration withLastRunTimeFieldName(String lastRunTimeFieldName)
{
this.lastRunTimeFieldName = lastRunTimeFieldName;
return (this);
}
/*******************************************************************************
** Getter for hoursBackForInitialTimestamp
**
*******************************************************************************/
public Integer getHoursBackForInitialTimestamp()
{
return hoursBackForInitialTimestamp;
}
/*******************************************************************************
** Setter for hoursBackForInitialTimestamp
**
*******************************************************************************/
public void setHoursBackForInitialTimestamp(Integer hoursBackForInitialTimestamp)
{
this.hoursBackForInitialTimestamp = hoursBackForInitialTimestamp;
}
/*******************************************************************************
** Fluent setter for hoursBackForInitialTimestamp
**
*******************************************************************************/
public BasepullConfiguration withHoursBackForInitialTimestamp(Integer hoursBackForInitialTimestamp)
{
this.hoursBackForInitialTimestamp = hoursBackForInitialTimestamp;
return (this);
}
/*******************************************************************************
** Getter for timestampField
**
*******************************************************************************/
public String getTimestampField()
{
return timestampField;
}
/*******************************************************************************
** Setter for timestampField
**
*******************************************************************************/
public void setTimestampField(String timestampField)
{
this.timestampField = timestampField;
}
/*******************************************************************************
** Fluent setter for timestampField
**
*******************************************************************************/
public BasepullConfiguration withTimestampField(String timestampField)
{
this.timestampField = timestampField;
return (this);
}
}

82
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java Normal file
View File

@ -0,0 +1,82 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
/*******************************************************************************
** Version of ExtractViaQueryStep that knows how to set up a basepull query.
*******************************************************************************/
public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
{
/*******************************************************************************
**
*******************************************************************************/
protected QQueryFilter getQueryFilter(RunBackendStepInput runBackendStepInput) throws QException
{
//////////////////////////////////////////////////////////////
// get input query filter or if not found, create a new one //
//////////////////////////////////////////////////////////////
QQueryFilter queryFilter = new QQueryFilter();
try
{
queryFilter = super.getQueryFilter(runBackendStepInput);
}
catch(QException qe)
{
///////////////////////////////////////////////////////////////////////////////////////
// if we catch here, assume that is because there was no default filter, continue on //
///////////////////////////////////////////////////////////////////////////////////////
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// build up a query filter that is against the source table for the given source table timestamp //
// field, finding any records that need processed. //
// query will be for: timestamp > lastRun AND timestamp <= thisRun. //
// then thisRun will be stored, so the next run shouldn't find any records from thisRun. //
///////////////////////////////////////////////////////////////////////////////////////////////////
queryFilter.addCriteria(new QFilterCriteria()
.withFieldName(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD))
.withOperator(QCriteriaOperator.GREATER_THAN)
.withValues(List.of(runBackendStepInput.getBasepullLastRunTime())));
queryFilter.addCriteria(new QFilterCriteria()
.withFieldName(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD))
.withOperator(QCriteriaOperator.LESS_THAN_OR_EQUALS)
.withValues(List.of(runBackendStepInput.getValueInstant(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY))));
queryFilter.addOrderBy(new QFilterOrderBy(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD)));
return (queryFilter);
}
}

7
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java
View File

@ -680,10 +680,13 @@ class QInstanceValidatorTest
@Test
void testAppSectionsMissingLabel()
{
///////////////////////////////////////////////////////////////////////////////////
// the enricher makes a label from the name, so, we'll just make them both null. //
///////////////////////////////////////////////////////////////////////////////////
QAppMetaData app = new QAppMetaData().withName("test")
.withChild(new QTableMetaData().withName("test"))
.withSection(new QAppSection("Section 1", null, new QIcon("person"), List.of("test"), null, null));
assertValidationFailureReasons((qInstance) -> qInstance.addApp(app), "Missing a label");
.withSection(new QAppSection(null, null, new QIcon("person"), List.of("test"), null, null));
assertValidationFailureReasons((qInstance) -> qInstance.addApp(app), "Missing a label", "Missing a name");
}

74
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStepTest.java Normal file
View File

@ -0,0 +1,74 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for ExtractViaBasepullQueryStep
*******************************************************************************/
class ExtractViaBasepullQueryStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
Instant timestamp = Instant.parse("1980-05-31T15:36:00Z");
Instant now = Instant.now();
RunBackendStepInput input = new RunBackendStepInput(TestUtils.defineInstance());
input.addValue(RunProcessAction.BASEPULL_TIMESTAMP_FIELD, "createDate");
input.addValue(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY, now);
input.setBasepullLastRunTime(timestamp);
QQueryFilter queryFilter = new ExtractViaBasepullQueryStep().getQueryFilter(input);
System.out.println(queryFilter);
assertEquals(2, queryFilter.getCriteria().size());
assertEquals("createDate", queryFilter.getCriteria().get(0).getFieldName());
assertEquals(QCriteriaOperator.GREATER_THAN, queryFilter.getCriteria().get(0).getOperator());
assertEquals(timestamp, queryFilter.getCriteria().get(0).getValues().get(0));
assertEquals("createDate", queryFilter.getCriteria().get(1).getFieldName());
assertEquals(QCriteriaOperator.LESS_THAN_OR_EQUALS, queryFilter.getCriteria().get(1).getOperator());
assertEquals(now, queryFilter.getCriteria().get(1).getValues().get(0));
assertEquals(1, queryFilter.getOrderBys().size());
assertEquals("createDate", queryFilter.getOrderBys().get(0).getFieldName());
assertTrue(queryFilter.getOrderBys().get(0).getIsAscending());
}
}

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -87,9 +87,9 @@ import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationM
import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

100
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIQueryAction.java
View File

@ -22,13 +22,11 @@
package com.kingsrook.qqq.backend.module.api.actions;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
@ -55,33 +53,83 @@ public class APIQueryAction extends AbstractAPIAction implements QueryInterface
QTableMetaData table = queryInput.getTable();
preAction(queryInput);
try
QueryOutput queryOutput = new QueryOutput(queryInput);
Integer originalLimit = queryInput.getLimit();
Integer limit = originalLimit;
Integer skip = queryInput.getSkip();
if(limit == null)
{
QQueryFilter filter = queryInput.getFilter();
String paramString = apiActionUtil.buildQueryStringForGet(filter, queryInput.getLimit(), queryInput.getSkip(), table.getFields());
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
HttpClient client = httpClientBuilder.build();
String url = apiActionUtil.buildTableUrl(table) + paramString;
LOG.info("API URL: " + url);
HttpGet request = new HttpGet(url);
apiActionUtil.setupAuthorizationInRequest(request);
apiActionUtil.setupContentTypeInRequest(request);
apiActionUtil.setupAdditionalHeaders(request);
HttpResponse response = client.execute(request);
List<QRecord> queryResults = apiActionUtil.processGetResponse(table, response);
QueryOutput queryOutput = new QueryOutput(queryInput);
queryOutput.addRecords(queryResults);
return (queryOutput);
limit = apiActionUtil.getApiStandardLimit();
}
catch(Exception e)
int totalCount = 0;
while(true)
{
LOG.warn("Error in API Query", e);
throw new QException("Error executing query: " + e.getMessage(), e);
try
{
QQueryFilter filter = queryInput.getFilter();
String paramString = apiActionUtil.buildQueryStringForGet(filter, limit, skip, table.getFields());
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
HttpClient client = httpClientBuilder.build();
String url = apiActionUtil.buildTableUrl(table) + paramString;
LOG.info("API URL: " + url);
///////////////////////////
// todo - 429 handling!! //
///////////////////////////
HttpGet request = new HttpGet(url);
apiActionUtil.setupAuthorizationInRequest(request);
apiActionUtil.setupContentTypeInRequest(request);
apiActionUtil.setupAdditionalHeaders(request);
HttpResponse response = client.execute(request);
int count = apiActionUtil.processGetResponse(table, response, queryOutput);
totalCount += count;
/////////////////////////////////////////////////////////////////////////
// if we've fetched at least as many as the original limit, then break //
/////////////////////////////////////////////////////////////////////////
if(originalLimit != null && totalCount >= originalLimit)
{
return (queryOutput);
}
////////////////////////////////////////////////////////////////////////////////////
// if we got back less than a full page this time, then we must be done, so break //
////////////////////////////////////////////////////////////////////////////////////
if(count == 0 || (limit != null && count < limit))
{
return (queryOutput);
}
///////////////////////////////////////////////////////////////////
// if there's an async callback that says we're cancelled, break //
///////////////////////////////////////////////////////////////////
if(queryInput.getAsyncJobCallback().wasCancelRequested())
{
LOG.info("Breaking query job, as requested.");
return (queryOutput);
}
////////////////////////////////////////////////////////////////////////////
// else, increment the skip by the count we just got, and query for more. //
////////////////////////////////////////////////////////////////////////////
if(skip == null)
{
skip = 0;
}
skip += count;
}
catch(Exception e)
{
LOG.warn("Error in API Query", e);
throw new QException("Error executing query: " + e.getMessage(), e);
}
}
}

41
qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java
View File

@ -26,7 +26,6 @@ import java.io.IOException;
import java.io.Serializable;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
@ -35,6 +34,8 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -74,7 +75,7 @@ public class BaseAPIActionUtil
*******************************************************************************/
public long getMillisToSleepAfterEveryCall()
{
return 0;
return (0);
}
@ -84,7 +85,7 @@ public class BaseAPIActionUtil
*******************************************************************************/
public int getInitialRateLimitBackoffMillis()
{
return 0;
return (0);
}
@ -94,7 +95,17 @@ public class BaseAPIActionUtil
*******************************************************************************/
public int getMaxAllowedRateLimitErrors()
{
return 0;
return (0);
}
/*******************************************************************************
**
*******************************************************************************/
public Integer getApiStandardLimit()
{
return (20);
}
@ -272,7 +283,7 @@ public class BaseAPIActionUtil
/*******************************************************************************
**
*******************************************************************************/
protected List<QRecord> processGetResponse(QTableMetaData table, HttpResponse response) throws IOException
protected int processGetResponse(QTableMetaData table, HttpResponse response, QueryOutput queryOutput) throws IOException
{
int statusCode = response.getStatusLine().getStatusCode();
System.out.println(statusCode);
@ -285,7 +296,7 @@ public class BaseAPIActionUtil
HttpEntity entity = response.getEntity();
String resultString = EntityUtils.toString(entity);
List<QRecord> recordList = new ArrayList<>();
int count = 0;
if(StringUtils.hasContent(resultString) && !resultString.equals("null"))
{
JSONArray resultList = null;
@ -309,16 +320,18 @@ public class BaseAPIActionUtil
{
for(int i = 0; i < resultList.length(); i++)
{
recordList.add(jsonObjectToRecord(resultList.getJSONObject(i), table.getFields()));
queryOutput.addRecord(jsonObjectToRecord(resultList.getJSONObject(i), table.getFields()));
count++;
}
}
else
{
recordList.add(jsonObjectToRecord(jsonObject, table.getFields()));
queryOutput.addRecord(jsonObjectToRecord(jsonObject, table.getFields()));
count++;
}
}
return (recordList);
return (count);
}
@ -483,8 +496,14 @@ public class BaseAPIActionUtil
*******************************************************************************/
public Integer processGetResponseForCount(QTableMetaData table, HttpResponse response) throws IOException
{
List<QRecord> queryResults = processGetResponse(table, response);
return (queryResults.size());
/////////////////////////////////////////////////////////////////////////////////////////
// set up a query output with a blank query input - e.g., one that isn't using a pipe. //
/////////////////////////////////////////////////////////////////////////////////////////
QueryOutput queryOutput = new QueryOutput(new QueryInput());
processGetResponse(table, response, queryOutput);
List<QRecord> records = queryOutput.getRecords();
return (records == null ? null : records.size());
}
}

6
qqq-dev-tools/bin/reset-snapshot-deps.sh
View File

@ -9,5 +9,9 @@ CURRENT_VERSION="$(cat $QQQ_DEV_TOOLS_DIR/CURRENT-SNAPSHOT-VERSION)"
MODULE_LIST_FILE=$QQQ_DEV_TOOLS_DIR/MODULE_LIST
for artifact in $(cat $MODULE_LIST_FILE); do
update-dep.sh $artifact ${CURRENT_VERSION}-SNAPSHOT
update-dep.sh $artifact ${CURRENT_VERSION}-SNAPSHOT -q
done
echo
echo git diff pom.xml
git diff pom.xml

10
qqq-dev-tools/bin/update-all-qqq-deps.sh
View File

@ -9,6 +9,14 @@ CURRENT_VERSION="$(cat $QQQ_DEV_TOOLS_DIR/CURRENT-SNAPSHOT-VERSION)"
MODULE_LIST_FILE=$QQQ_DEV_TOOLS_DIR/MODULE_LIST
for module in $(cat $MODULE_LIST_FILE); do
echo "Updating $module..."
version=$(get-latest-snapshot.sh $module $CURRENT_VERSION)
update-dep.sh $module $version
update-dep.sh $module $version -q
done
echo
echo git diff pom.xml
git diff pom.xml
newVersion=$(grep -A1 qqq-backend-core pom.xml | tail -1 | sed 's/.*<version>//;s/<\/version>.*//;s/-........\......./ snapshot/')
echo "You might want to commit that with:"
echo " git commit -m \"Updated qqq deps to $newVersion\" pom.xml"

14
qqq-dev-tools/bin/update-dep.sh
View File

@ -8,6 +8,11 @@
dep=$1
version=$2
verbose=1
if [ "$3" == "-q" ]; then
verbose=0;
fi
if [ -z "$dep" -o -z "$version" ]; then
echo "What dependency?"
@ -38,7 +43,12 @@ if [ $lineNo -lt $dependenciesTagLineNo ]; then
exit 0;
fi
echo "Going to update version of $dep at line $lineNo"
if [ "$verbose" == "1" ]; then
echo "Going to update version of $dep at line $lineNo"
fi
gsed -i "${lineNo}s/<version>.*</<version>$version</" pom.xml
git diff pom.xml
if [ "$verbose" == "1" ]; then
git diff pom.xml
fi