QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-20 06:00:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ExtractViaBasepullQueryStep; add pagination & piping to api query

Browse Source
This commit is contained in:
darin.kelkhoff
2022-10-26 12:24:35 -05:00
parent 77927fd318
commit 82810c2b66
18 changed files with 689 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java
View File

@ -52,7 +52,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
import com.kingsrook.qqq.backend.core.state.StateType;
@ -69,9 +69,11 @@ import org.apache.logging.log4j.Logger;
*******************************************************************************/
public class RunProcessAction
{
private static final Logger LOG = LogManager.getLogger(RunProcessAction.class);
public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
private static final Logger LOG = LogManager.getLogger(RunProcessAction.class);
public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
public static final String BASEPULL_TIMESTAMP_FIELD = "basepullTimestampField";
@ -422,7 +424,7 @@ public class RunProcessAction
/*******************************************************************************
**
** Insert or update the last runtime value for this basepull into the backend.
*******************************************************************************/
protected void storeLastRunTime(RunProcessInput runProcessInput, QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
{
@ -489,13 +491,22 @@ public class RunProcessAction
/*******************************************************************************
**
** Lookup the last runtime for this basepull, and set it (plus now) in the process's
** values.
*******************************************************************************/
protected void persistLastRunTime(RunProcessInput runProcessInput, QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
{
////////////////////////////////////////////////////////////////////////////////////////////////
// store 'now', which will be used to update basepull record if process completes sucessfully //
////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////
// if these values were already computed, don't re-do //
////////////////////////////////////////////////////////
if(runProcessInput.getValue(BASEPULL_THIS_RUNTIME_KEY) != null)
{
return;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// store 'now', which will be used to update basepull record if process completes successfully //
/////////////////////////////////////////////////////////////////////////////////////////////////
Instant now = Instant.now();
runProcessInput.getValues().put(BASEPULL_THIS_RUNTIME_KEY, now);
@ -533,5 +544,6 @@ public class RunProcessAction
}
runProcessInput.getValues().put(BASEPULL_LAST_RUNTIME_KEY, lastRunTime);
runProcessInput.getValues().put(BASEPULL_TIMESTAMP_FIELD, basepullConfiguration.getTimestampField());
}
}

18
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -316,6 +316,24 @@ public class QInstanceEnricher
{
generateAppSections(app);
}
for(QAppSection section : CollectionUtils.nonNullList(app.getSections()))
{
enrichAppSection(section);
}
}
/*******************************************************************************
**
*******************************************************************************/
private void enrichAppSection(QAppSection section)
{
if(!StringUtils.hasContent(section.getLabel()))
{
section.setLabel(nameToLabel(section.getName()));
}
}

11
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java
View File

@ -419,6 +419,17 @@ public class RunBackendStepInput extends AbstractActionInput
/*******************************************************************************
** Getter for a single field's value
**
*******************************************************************************/
public Instant getValueInstant(String fieldName)
{
return (ValueUtils.getValueAsInstant(getValue(fieldName)));
}
/*******************************************************************************
** Accessor for processState - protected, because we generally want to access
** its members through wrapper methods, we think

47
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java
View File

@ -25,6 +25,9 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
@ -33,6 +36,8 @@ import java.util.List;
*******************************************************************************/
public class QFilterCriteria implements Serializable, Cloneable
{
private static final Logger LOG = LogManager.getLogger(QFilterCriteria.class);
private String fieldName;
private QCriteriaOperator operator;
private List<Serializable> values;
@ -183,4 +188,46 @@ public class QFilterCriteria implements Serializable, Cloneable
this.values = values;
return this;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
StringBuilder rs = new StringBuilder(fieldName);
try
{
rs.append(" ").append(operator).append(" ");
if(CollectionUtils.nullSafeHasContents(values))
{
if(values.size() == 1)
{
rs.append(values.get(0));
}
else
{
int index = 0;
for(Serializable value : values)
{
if(index++ > 9)
{
rs.append("and ").append(values.size() - index).append(" more");
break;
}
rs.append(value).append(",");
}
}
}
}
catch(Exception e)
{
LOG.warn("Error in toString", e);
rs.append("Error generating toString...");
}
return (rs.toString());
}
}

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterOrderBy.java
View File

@ -151,4 +151,14 @@ public class QFilterOrderBy implements Serializable, Cloneable
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
return (fieldName + " " + (isAscending ? "ASC" : "DESC"));
}
}

41
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java
View File

@ -26,6 +26,8 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
@ -34,6 +36,8 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
*******************************************************************************/
public class QQueryFilter implements Serializable, Cloneable
{
private static final Logger LOG = LogManager.getLogger(QQueryFilter.class);
private List<QFilterCriteria> criteria = new ArrayList<>();
private List<QFilterOrderBy> orderBys = new ArrayList<>();
@ -301,4 +305,41 @@ public class QQueryFilter implements Serializable, Cloneable
subFilters.add(subFilter);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
StringBuilder rs = new StringBuilder("(");
try
{
for(QFilterCriteria criterion : CollectionUtils.nonNullList(criteria))
{
rs.append(criterion).append(" ").append(getBooleanOperator());
}
for(QQueryFilter subFilter : CollectionUtils.nonNullList(subFilters))
{
rs.append(subFilter);
}
rs.append(")");
rs.append("OrderBy[");
for(QFilterOrderBy orderBy : orderBys)
{
rs.append(orderBy).append(",");
}
rs.append("]");
}
catch(Exception e)
{
LOG.warn("Error in toString", e);
rs.append("Error generating toString...");
}
return (rs.toString());
}
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java
View File

@ -33,7 +33,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
/*******************************************************************************

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java
View File

@ -822,4 +822,5 @@ public class QTableMetaData implements QAppChildMetaData, Serializable
return (this);
}
}

247
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/BasepullConfiguration.java Normal file
View File

@ -0,0 +1,247 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
import java.io.Serializable;
/*******************************************************************************
** Class for storing all basepull configuration data
**
*******************************************************************************/
public class BasepullConfiguration implements Serializable
{
private String tableName; // the table that stores the basepull timestamps
private String keyField; // the field in the basepull timestamps table that stores the key of the basepull (e.g., a process name)
private String keyValue; // the key applied to the keyField - optional - if not set, process.getName is used.
private String lastRunTimeFieldName; // the field in the basepull timestamps table that stores the last-run time for the job.
private Integer hoursBackForInitialTimestamp; // for the first-run use-case (where there is no row in the timestamps table), how many hours back in time to look.
private String timestampField; // the name of the field in the table being queried against the last-run timestamp.
/*******************************************************************************
** Getter for tableName
**
*******************************************************************************/
public String getTableName()
{
return tableName;
}
/*******************************************************************************
** Setter for tableName
**
*******************************************************************************/
public void setTableName(String tableName)
{
this.tableName = tableName;
}
/*******************************************************************************
** Fluent setter for tableName
**
*******************************************************************************/
public BasepullConfiguration withTableName(String tableName)
{
this.tableName = tableName;
return (this);
}
/*******************************************************************************
** Getter for keyField
**
*******************************************************************************/
public String getKeyField()
{
return keyField;
}
/*******************************************************************************
** Setter for keyField
**
*******************************************************************************/
public void setKeyField(String keyField)
{
this.keyField = keyField;
}
/*******************************************************************************
** Fluent setter for keyField
**
*******************************************************************************/
public BasepullConfiguration withKeyField(String keyField)
{
this.keyField = keyField;
return (this);
}
/*******************************************************************************
** Getter for keyValue
**
*******************************************************************************/
public String getKeyValue()
{
return keyValue;
}
/*******************************************************************************
** Setter for keyValue
**
*******************************************************************************/
public void setKeyValue(String keyValue)
{
this.keyValue = keyValue;
}
/*******************************************************************************
** Fluent setter for keyValue
**
*******************************************************************************/
public BasepullConfiguration withKeyValue(String keyValue)
{
this.keyValue = keyValue;
return (this);
}
/*******************************************************************************
** Getter for lastRunTimeFieldName
**
*******************************************************************************/
public String getLastRunTimeFieldName()
{
return lastRunTimeFieldName;
}
/*******************************************************************************
** Setter for lastRunTimeFieldName
**
*******************************************************************************/
public void setLastRunTimeFieldName(String lastRunTimeFieldName)
{
this.lastRunTimeFieldName = lastRunTimeFieldName;
}
/*******************************************************************************
** Fluent setter for lastRunTimeFieldName
**
*******************************************************************************/
public BasepullConfiguration withLastRunTimeFieldName(String lastRunTimeFieldName)
{
this.lastRunTimeFieldName = lastRunTimeFieldName;
return (this);
}
/*******************************************************************************
** Getter for hoursBackForInitialTimestamp
**
*******************************************************************************/
public Integer getHoursBackForInitialTimestamp()
{
return hoursBackForInitialTimestamp;
}
/*******************************************************************************
** Setter for hoursBackForInitialTimestamp
**
*******************************************************************************/
public void setHoursBackForInitialTimestamp(Integer hoursBackForInitialTimestamp)
{
this.hoursBackForInitialTimestamp = hoursBackForInitialTimestamp;
}
/*******************************************************************************
** Fluent setter for hoursBackForInitialTimestamp
**
*******************************************************************************/
public BasepullConfiguration withHoursBackForInitialTimestamp(Integer hoursBackForInitialTimestamp)
{
this.hoursBackForInitialTimestamp = hoursBackForInitialTimestamp;
return (this);
}
/*******************************************************************************
** Getter for timestampField
**
*******************************************************************************/
public String getTimestampField()
{
return timestampField;
}
/*******************************************************************************
** Setter for timestampField
**
*******************************************************************************/
public void setTimestampField(String timestampField)
{
this.timestampField = timestampField;
}
/*******************************************************************************
** Fluent setter for timestampField
**
*******************************************************************************/
public BasepullConfiguration withTimestampField(String timestampField)
{
this.timestampField = timestampField;
return (this);
}
}

82
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java Normal file
View File

@ -0,0 +1,82 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
/*******************************************************************************
** Version of ExtractViaQueryStep that knows how to set up a basepull query.
*******************************************************************************/
public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
{
/*******************************************************************************
**
*******************************************************************************/
protected QQueryFilter getQueryFilter(RunBackendStepInput runBackendStepInput) throws QException
{
//////////////////////////////////////////////////////////////
// get input query filter or if not found, create a new one //
//////////////////////////////////////////////////////////////
QQueryFilter queryFilter = new QQueryFilter();
try
{
queryFilter = super.getQueryFilter(runBackendStepInput);
}
catch(QException qe)
{
///////////////////////////////////////////////////////////////////////////////////////
// if we catch here, assume that is because there was no default filter, continue on //
///////////////////////////////////////////////////////////////////////////////////////
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// build up a query filter that is against the source table for the given source table timestamp //
// field, finding any records that need processed. //
// query will be for: timestamp > lastRun AND timestamp <= thisRun. //
// then thisRun will be stored, so the next run shouldn't find any records from thisRun. //
///////////////////////////////////////////////////////////////////////////////////////////////////
queryFilter.addCriteria(new QFilterCriteria()
.withFieldName(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD))
.withOperator(QCriteriaOperator.GREATER_THAN)
.withValues(List.of(runBackendStepInput.getBasepullLastRunTime())));
queryFilter.addCriteria(new QFilterCriteria()
.withFieldName(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD))
.withOperator(QCriteriaOperator.LESS_THAN_OR_EQUALS)
.withValues(List.of(runBackendStepInput.getValueInstant(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY))));
queryFilter.addOrderBy(new QFilterOrderBy(runBackendStepInput.getValueString(RunProcessAction.BASEPULL_TIMESTAMP_FIELD)));
return (queryFilter);
}
}