QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-20 06:00:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

sprint-14: initial checkin of basepull capability on processes

Browse Source
This commit is contained in:
Tim Chamberlain
2022-10-24 17:06:11 -05:00
parent 44537e182d
commit 128c379f10
11 changed files with 658 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java
View File

@ -23,26 +23,42 @@ package com.kingsrook.qqq.backend.core.actions.processes;
import java.io.Serializable;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration;
import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
import com.kingsrook.qqq.backend.core.state.StateType;
import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -53,7 +69,9 @@ import org.apache.logging.log4j.Logger;
*******************************************************************************/
public class RunProcessAction
{
private static final Logger LOG = LogManager.getLogger(RunProcessAction.class);
private static final Logger LOG = LogManager.getLogger(RunProcessAction.class);
public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
@ -84,6 +102,18 @@ public class RunProcessAction
UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.fromString(runProcessInput.getProcessUUID()), StateType.PROCESS_STATUS);
ProcessState processState = primeProcessState(runProcessInput, stateKey, process);
/////////////////////////////////////////////////////////
// if process is 'basepull' style, keep track of 'now' //
/////////////////////////////////////////////////////////
BasepullConfiguration basepullConfiguration = process.getBasepullConfiguration();
if(basepullConfiguration != null)
{
///////////////////////////////////////
// get the stored basepull timestamp //
///////////////////////////////////////
persistLastRunTime(runProcessInput, process, basepullConfiguration);
}
try
{
String lastStepName = runProcessInput.getStartAfterStep();
@ -151,6 +181,17 @@ public class RunProcessAction
throw (new QException("Unsure how to run a step of type: " + step.getClass().getName()));
}
}
///////////////////////////////////////////////////////////////////////////////
// if 'basepull' style process, store the time stored before process was ran //
///////////////////////////////////////////////////////////////////////////////
if(basepullConfiguration != null)
{
///////////////////////////////////////
// get the stored basepull timestamp //
///////////////////////////////////////
storeLastRunTime(runProcessInput, process, basepullConfiguration);
}
}
catch(QException qe)
{
@ -250,7 +291,17 @@ public class RunProcessAction
runBackendStepInput.setTableName(process.getTableName());
runBackendStepInput.setSession(runProcessInput.getSession());
runBackendStepInput.setCallback(runProcessInput.getCallback());
runBackendStepInput.setFrontendStepBehavior(runProcessInput.getFrontendStepBehavior());
runBackendStepInput.setAsyncJobCallback(runProcessInput.getAsyncJobCallback());
///////////////////////////////////////////////////////////////
// if 'basepull' values are in the inputs, add to step input //
///////////////////////////////////////////////////////////////
if(runProcessInput.getValues().containsKey(BASEPULL_LAST_RUNTIME_KEY))
{
runBackendStepInput.setBasepullLastRunTime((Instant) runProcessInput.getValues().get(BASEPULL_LAST_RUNTIME_KEY));
}
RunBackendStepOutput lastFunctionResult = new RunBackendStepAction().execute(runBackendStepInput);
storeState(stateKey, lastFunctionResult.getProcessState());
@ -368,4 +419,119 @@ public class RunProcessAction
return (getStateProvider().get(ProcessState.class, stateKey));
}
/*******************************************************************************
**
*******************************************************************************/
protected void storeLastRunTime(RunProcessInput runProcessInput, QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
{
String basepullTableName = basepullConfiguration.getTableName();
String basepullKeyFieldName = basepullConfiguration.getKeyField();
String basepullLastRunTimeFieldName = basepullConfiguration.getLastRunTimeFieldName();
String basepullKeyValue = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
///////////////////////////////////////
// get the stored basepull timestamp //
///////////////////////////////////////
QueryInput queryInput = new QueryInput(runProcessInput.getInstance());
queryInput.setSession(runProcessInput.getSession());
queryInput.setTableName(basepullTableName);
queryInput.setFilter(new QQueryFilter().withCriteria(
new QFilterCriteria()
.withFieldName(basepullKeyFieldName)
.withOperator(QCriteriaOperator.EQUALS)
.withValues(List.of(basepullKeyValue))));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
//////////////////////////////////////////
// get the runtime for this process run //
//////////////////////////////////////////
Instant newRunTime = (Instant) runProcessInput.getValues().get(BASEPULL_THIS_RUNTIME_KEY);
/////////////////////////////////////////////////
// update if found, otherwise insert new value //
/////////////////////////////////////////////////
if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
{
///////////////////////////////////////////////////////////////////////////////
// update the basepull table with 'now' (which is before original query ran) //
///////////////////////////////////////////////////////////////////////////////
QRecord basepullRecord = queryOutput.getRecords().get(0);
basepullRecord.setValue(basepullLastRunTimeFieldName, newRunTime);
////////////
// update //
////////////
UpdateInput updateInput = new UpdateInput(runProcessInput.getInstance());
updateInput.setSession(runProcessInput.getSession());
updateInput.setTableName(basepullTableName);
updateInput.setRecords(List.of(basepullRecord));
new UpdateAction().execute(updateInput);
}
else
{
QRecord basepullRecord = new QRecord()
.withValue(basepullKeyFieldName, basepullKeyValue)
.withValue(basepullLastRunTimeFieldName, newRunTime);
////////////////////////////////
// insert new basepull record //
////////////////////////////////
InsertInput insertInput = new InsertInput(runProcessInput.getInstance());
insertInput.setSession(runProcessInput.getSession());
insertInput.setTableName(basepullTableName);
insertInput.setRecords(List.of(basepullRecord));
new InsertAction().execute(insertInput);
}
}
/*******************************************************************************
**
*******************************************************************************/
protected void persistLastRunTime(RunProcessInput runProcessInput, QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
{
////////////////////////////////////////////////////////////////////////////////////////////////
// store 'now', which will be used to update basepull record if process completes sucessfully //
////////////////////////////////////////////////////////////////////////////////////////////////
Instant now = Instant.now();
runProcessInput.getValues().put(BASEPULL_THIS_RUNTIME_KEY, now);
String basepullTableName = basepullConfiguration.getTableName();
String basepullKeyFieldName = basepullConfiguration.getKeyField();
String basepullLastRunTimeFieldName = basepullConfiguration.getLastRunTimeFieldName();
Integer basepullHoursBackForInitialTimestamp = basepullConfiguration.getHoursBackForInitialTimestamp();
String basepullKeyValue = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
///////////////////////////////////////
// get the stored basepull timestamp //
///////////////////////////////////////
QueryInput queryInput = new QueryInput(runProcessInput.getInstance());
queryInput.setSession(runProcessInput.getSession());
queryInput.setTableName(basepullTableName);
queryInput.setFilter(new QQueryFilter().withCriteria(
new QFilterCriteria()
.withFieldName(basepullKeyFieldName)
.withOperator(QCriteriaOperator.EQUALS)
.withValues(List.of(basepullKeyValue))));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
///////////////////////////////////////////////////////////////////////////////////////////////////
// get the stored time, if not, default to 'now' unless a number of hours to offset was provided //
///////////////////////////////////////////////////////////////////////////////////////////////////
Instant lastRunTime = now;
if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
{
QRecord basepullRecord = queryOutput.getRecords().get(0);
lastRunTime = ValueUtils.getValueAsInstant(basepullRecord.getValue(basepullLastRunTimeFieldName));
}
else if(basepullHoursBackForInitialTimestamp != null)
{
lastRunTime = lastRunTime.minus(basepullHoursBackForInitialTimestamp, ChronoUnit.HOURS);
}
runProcessInput.getValues().put(BASEPULL_LAST_RUNTIME_KEY, lastRunTime);
}
}

83
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.processes;
import java.io.Serializable;
import java.time.Instant;
import java.time.LocalDate;
import java.util.List;
import java.util.Map;
@ -44,12 +45,14 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
*******************************************************************************/
public class RunBackendStepInput extends AbstractActionInput
{
private ProcessState processState;
private String processName;
private String tableName;
private String stepName;
private QProcessCallback callback;
private AsyncJobCallback asyncJobCallback;
private ProcessState processState;
private String processName;
private String tableName;
private String stepName;
private QProcessCallback callback;
private AsyncJobCallback asyncJobCallback;
private RunProcessInput.FrontendStepBehavior frontendStepBehavior;
private Instant basepullLastRunTime;
////////////////////////////////////////////////////////////////////////////
// note - new fields should generally be added in method: cloneFieldsInto //
@ -453,4 +456,72 @@ public class RunBackendStepInput extends AbstractActionInput
return (asyncJobCallback);
}
/*******************************************************************************
** Getter for frontendStepBehavior
**
*******************************************************************************/
public RunProcessInput.FrontendStepBehavior getFrontendStepBehavior()
{
return frontendStepBehavior;
}
/*******************************************************************************
** Setter for frontendStepBehavior
**
*******************************************************************************/
public void setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior frontendStepBehavior)
{
this.frontendStepBehavior = frontendStepBehavior;
}
/*******************************************************************************
** Fluent setter for frontendStepBehavior
**
*******************************************************************************/
public RunBackendStepInput withFrontendStepBehavior(RunProcessInput.FrontendStepBehavior frontendStepBehavior)
{
this.frontendStepBehavior = frontendStepBehavior;
return (this);
}
/*******************************************************************************
** Getter for basepullLastRunTime
**
*******************************************************************************/
public Instant getBasepullLastRunTime()
{
return basepullLastRunTime;
}
/*******************************************************************************
** Setter for basepullLastRunTime
**
*******************************************************************************/
public void setBasepullLastRunTime(Instant basepullLastRunTime)
{
this.basepullLastRunTime = basepullLastRunTime;
}
/*******************************************************************************
** Fluent setter for basepullLastRunTime
**
*******************************************************************************/
public RunBackendStepInput withBasepullLastRunTime(Instant basepullLastRunTime)
{
this.basepullLastRunTime = basepullLastRunTime;
return (this);
}
}

44
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java
View File

@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.general.BasepullConfiguration;
/*******************************************************************************
@ -41,10 +42,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaDa
*******************************************************************************/
public class QProcessMetaData implements QAppChildMetaData
{
private String name;
private String label;
private String tableName;
private boolean isHidden = false;
private String name;
private String label;
private String tableName;
private boolean isHidden = false;
private BasepullConfiguration basepullConfiguration;
private List<QStepMetaData> stepList; // these are the steps that are ran, by-default, in the order they are ran in
private Map<String, QStepMetaData> steps; // this is the full map of possible steps
@ -473,4 +475,38 @@ public class QProcessMetaData implements QAppChildMetaData
return (this);
}
/*******************************************************************************
** Getter for basepullConfiguration
**
*******************************************************************************/
public BasepullConfiguration getBasepullConfiguration()
{
return basepullConfiguration;
}
/*******************************************************************************
** Setter for basepullConfiguration
**
*******************************************************************************/
public void setBasepullConfiguration(BasepullConfiguration basepullConfiguration)
{
this.basepullConfiguration = basepullConfiguration;
}
/*******************************************************************************
** Fluent setter for basepullConfiguration
**
*******************************************************************************/
public QProcessMetaData withBasepullConfiguration(BasepullConfiguration basepullConfiguration)
{
this.basepullConfiguration = basepullConfiguration;
return (this);
}
}

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java
View File

@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
import org.apache.logging.log4j.LogManager;
@ -66,6 +67,12 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
return;
}
if(runBackendStepInput.getFrontendStepBehavior() != null && runBackendStepInput.getFrontendStepBehavior().equals(RunProcessInput.FrontendStepBehavior.SKIP))
{
LOG.info("Skipping preview because frontent behavior is [" + RunProcessInput.FrontendStepBehavior.SKIP + "].");
return;
}
/////////////////////////////////////////////////////////////////
// if we're running inside an automation, then skip this step. //
/////////////////////////////////////////////////////////////////
@ -140,7 +147,6 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
/*******************************************************************************
**
*******************************************************************************/
@ -154,7 +160,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
///////////////////////////////////////////////////////////////////////
// make streamed input & output objects from the run input & outputs //
///////////////////////////////////////////////////////////////////////
StreamedBackendStepInput streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, qRecords);
StreamedBackendStepInput streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, qRecords);
StreamedBackendStepOutput streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput);
/////////////////////////////////////////////////////

211
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/BasepullConfiguration.java Normal file
View File

@ -0,0 +1,211 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.general;
import java.io.Serializable;
/*******************************************************************************
** Class for storing all basepull configuration data
**
*******************************************************************************/
public class BasepullConfiguration implements Serializable
{
private String tableName;
private String keyField;
private String keyValue;
private String lastRunTimeFieldName;
private Integer hoursBackForInitialTimestamp;
/*******************************************************************************
** Getter for tableName
**
*******************************************************************************/
public String getTableName()
{
return tableName;
}
/*******************************************************************************
** Setter for tableName
**
*******************************************************************************/
public void setTableName(String tableName)
{
this.tableName = tableName;
}
/*******************************************************************************
** Fluent setter for tableName
**
*******************************************************************************/
public BasepullConfiguration withTableName(String tableName)
{
this.tableName = tableName;
return (this);
}
/*******************************************************************************
** Getter for keyField
**
*******************************************************************************/
public String getKeyField()
{
return keyField;
}
/*******************************************************************************
** Setter for keyField
**
*******************************************************************************/
public void setKeyField(String keyField)
{
this.keyField = keyField;
}
/*******************************************************************************
** Fluent setter for keyField
**
*******************************************************************************/
public BasepullConfiguration withKeyField(String keyField)
{
this.keyField = keyField;
return (this);
}
/*******************************************************************************
** Getter for keyValue
**
*******************************************************************************/
public String getKeyValue()
{
return keyValue;
}
/*******************************************************************************
** Setter for keyValue
**
*******************************************************************************/
public void setKeyValue(String keyValue)
{
this.keyValue = keyValue;
}
/*******************************************************************************
** Fluent setter for keyValue
**
*******************************************************************************/
public BasepullConfiguration withKeyValue(String keyValue)
{
this.keyValue = keyValue;
return (this);
}
/*******************************************************************************
** Getter for lastRunTimeFieldName
**
*******************************************************************************/
public String getLastRunTimeFieldName()
{
return lastRunTimeFieldName;
}
/*******************************************************************************
** Setter for lastRunTimeFieldName
**
*******************************************************************************/
public void setLastRunTimeFieldName(String lastRunTimeFieldName)
{
this.lastRunTimeFieldName = lastRunTimeFieldName;
}
/*******************************************************************************
** Fluent setter for lastRunTimeFieldName
**
*******************************************************************************/
public BasepullConfiguration withLastRunTimeFieldName(String lastRunTimeFieldName)
{
this.lastRunTimeFieldName = lastRunTimeFieldName;
return (this);
}
/*******************************************************************************
** Getter for hoursBackForInitialTimestamp
**
*******************************************************************************/
public Integer getHoursBackForInitialTimestamp()
{
return hoursBackForInitialTimestamp;
}
/*******************************************************************************
** Setter for hoursBackForInitialTimestamp
**
*******************************************************************************/
public void setHoursBackForInitialTimestamp(Integer hoursBackForInitialTimestamp)
{
this.hoursBackForInitialTimestamp = hoursBackForInitialTimestamp;
}
/*******************************************************************************
** Fluent setter for hoursBackForInitialTimestamp
**
*******************************************************************************/
public BasepullConfiguration withHoursBackForInitialTimestamp(Integer hoursBackForInitialTimestamp)
{
this.hoursBackForInitialTimestamp = hoursBackForInitialTimestamp;
return (this);
}
}