QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Checkpoint, WIP on processes in api - mostly done

Browse Source
This commit is contained in:
darin.kelkhoff
2023-06-14 11:53:43 -05:00
parent eee7354e77
commit 19ee5bcb23
32 changed files with 1875 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java
View File

@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
import com.kingsrook.qqq.backend.core.state.StateType;
import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.logging.log4j.Level;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -50,6 +51,7 @@ public class AsyncJobManager
{
private static final QLogger LOG = QLogger.getLogger(AsyncJobManager.class);
private String forcedJobUUID = null;
/*******************************************************************************
@ -69,7 +71,8 @@ public class AsyncJobManager
*******************************************************************************/
public <T extends Serializable> T startJob(String jobName, long timeout, TimeUnit timeUnit, AsyncJob<T> asyncJob) throws JobGoingAsyncException, QException
{
UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.ASYNC_JOB_STATUS);
UUID jobUUID = StringUtils.hasContent(forcedJobUUID) ? UUID.fromString(forcedJobUUID) : UUID.randomUUID();
UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(jobUUID, StateType.ASYNC_JOB_STATUS);
AsyncJobStatus asyncJobStatus = new AsyncJobStatus();
asyncJobStatus.setState(AsyncJobState.RUNNING);
getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
@ -205,4 +208,35 @@ public class AsyncJobManager
jobStatus.ifPresent(asyncJobStatus -> asyncJobStatus.setCancelRequested(true));
}
/*******************************************************************************
** Getter for forcedJobUUID
*******************************************************************************/
public String getForcedJobUUID()
{
return (this.forcedJobUUID);
}
/*******************************************************************************
** Setter for forcedJobUUID
*******************************************************************************/
public void setForcedJobUUID(String forcedJobUUID)
{
this.forcedJobUUID = forcedJobUUID;
}
/*******************************************************************************
** Fluent setter for forcedJobUUID
*******************************************************************************/
public AsyncJobManager withForcedJobUUID(String forcedJobUUID)
{
this.forcedJobUUID = forcedJobUUID;
return (this);
}
}

10
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
View File

@ -63,6 +63,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QSupplementalProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
@ -1226,6 +1227,11 @@ public class QInstanceValidator
}
}
for(QSupplementalProcessMetaData supplementalProcessMetaData : CollectionUtils.nonNullMap(process.getSupplementalMetaData()).values())
{
supplementalProcessMetaData.validate(qInstance, process, this);
}
});
}
}
@ -1703,7 +1709,7 @@ public class QInstanceValidator
** But if it throws, add the provided message to the list of errors (and return false,
** e.g., in case you need to stop evaluating rules to avoid exceptions).
*******************************************************************************/
private boolean assertNoException(UnsafeLambda unsafeLambda, String message)
public boolean assertNoException(UnsafeLambda unsafeLambda, String message)
{
try
{
@ -1736,7 +1742,7 @@ public class QInstanceValidator
/*******************************************************************************
**
*******************************************************************************/
private void warn(String message)
public void warn(String message)
{
if(printWarnings)
{

31
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryFilterLink.java
View File

@ -22,8 +22,10 @@
package com.kingsrook.qqq.backend.core.model.actions.processes;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.logging.LogPair;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -76,6 +78,35 @@ public class ProcessSummaryFilterLink implements ProcessSummaryLineInterface
/*******************************************************************************
**
*******************************************************************************/
@JsonIgnore
public String getFullText()
{
StringBuilder rs = new StringBuilder();
if(StringUtils.hasContent(linkPreText))
{
rs.append(linkPreText).append(" ");
}
if(StringUtils.hasContent(linkText))
{
rs.append(linkText).append(" ");
}
if(StringUtils.hasContent(linkPostText))
{
rs.append(linkPostText).append(" ");
}
rs.deleteCharAt(rs.length() - 1);
return (rs.toString());
}
/*******************************************************************************
**
*******************************************************************************/

13
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java
View File

@ -26,6 +26,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.logging.LogPair;
import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -395,15 +396,19 @@ public class ProcessSummaryLine implements ProcessSummaryLineInterface
{
if(count != null)
{
String baseMessage;
if(count.equals(1))
{
setMessage((isPast ? getSingularPastMessage() : getSingularFutureMessage())
+ (messageSuffix == null ? "" : messageSuffix));
baseMessage = isPast ? getSingularPastMessage() : getSingularFutureMessage();
}
else
{
setMessage((isPast ? getPluralPastMessage() : getPluralFutureMessage())
+ (messageSuffix == null ? "" : messageSuffix));
baseMessage = isPast ? getPluralPastMessage() : getPluralFutureMessage();
}
if(StringUtils.hasContent(baseMessage))
{
setMessage(baseMessage + ObjectUtils.requireConditionElse(messageSuffix, StringUtils::hasContent, ""));
}
}
}

31
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryRecordLink.java
View File

@ -23,7 +23,9 @@ package com.kingsrook.qqq.backend.core.model.actions.processes;
import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.logging.LogPair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -64,6 +66,35 @@ public class ProcessSummaryRecordLink implements ProcessSummaryLineInterface
/*******************************************************************************
**
*******************************************************************************/
@JsonIgnore
public String getFullText()
{
StringBuilder rs = new StringBuilder();
if(StringUtils.hasContent(linkPreText))
{
rs.append(linkPreText).append(" ");
}
if(StringUtils.hasContent(linkText))
{
rs.append(linkText).append(" ");
}
if(StringUtils.hasContent(linkPostText))
{
rs.append(linkPostText).append(" ");
}
rs.deleteCharAt(rs.length() - 1);
return (rs.toString());
}
/*******************************************************************************
**
*******************************************************************************/

14
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QSupplementalProcessMetaData.java
View File

@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes;
import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
/*******************************************************************************
@ -75,4 +77,16 @@ public abstract class QSupplementalProcessMetaData
// noop in base class //
////////////////////////
}
/*******************************************************************************
**
*******************************************************************************/
public void validate(QInstance qInstance, QProcessMetaData process, QInstanceValidator qInstanceValidator)
{
////////////////////////
// noop in base class //
////////////////////////
}
}

39
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java
View File

@ -27,6 +27,7 @@ import java.util.List;
import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@ -71,11 +72,14 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
return;
}
if(runBackendStepInput.getFrontendStepBehavior() != null && runBackendStepInput.getFrontendStepBehavior().equals(RunProcessInput.FrontendStepBehavior.SKIP))
{
LOG.debug("Skipping preview because frontend behavior is [" + RunProcessInput.FrontendStepBehavior.SKIP + "].");
return;
}
//////////////////////////////
// set up the extract steps //
//////////////////////////////
AbstractExtractStep extractStep = getExtractStep(runBackendStepInput);
RecordPipe recordPipe = new RecordPipe();
extractStep.setLimit(limit);
extractStep.setRecordPipe(recordPipe);
extractStep.preRun(runBackendStepInput, runBackendStepOutput);
/////////////////////////////////////////////////////////////////
// if we're running inside an automation, then skip this step. //
@ -86,17 +90,26 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
return;
}
//////////////////////////////////////////
// set up the extract & transform steps //
//////////////////////////////////////////
AbstractExtractStep extractStep = getExtractStep(runBackendStepInput);
RecordPipe recordPipe = new RecordPipe();
extractStep.setLimit(limit);
extractStep.setRecordPipe(recordPipe);
extractStep.preRun(runBackendStepInput, runBackendStepOutput);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if skipping frontend steps, skip this action - //
// but, if inside an (ideally, only async) API call, at least do the count, so status calls can get x of y status //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(RunProcessInput.FrontendStepBehavior.SKIP.equals(runBackendStepInput.getFrontendStepBehavior()))
{
if(QContext.getQSession().getValue("apiVersion") != null)
{
countRecords(runBackendStepInput, runBackendStepOutput, extractStep);
}
LOG.debug("Skipping preview because frontend behavior is [" + RunProcessInput.FrontendStepBehavior.SKIP + "].");
return;
}
countRecords(runBackendStepInput, runBackendStepOutput, extractStep);
//////////////////////////////
// setup the transform step //
//////////////////////////////
AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
transformStep.preRun(runBackendStepInput, runBackendStepOutput);

37
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ExceptionUtils.java
View File

@ -22,7 +22,9 @@
package com.kingsrook.qqq.backend.core.utils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@ -88,4 +90,39 @@ public class ExceptionUtils
return (root);
}
/*******************************************************************************
**
*******************************************************************************/
public static String concatenateMessagesFromChain(Exception exception)
{
if(exception == null)
{
return (null);
}
List<String> messages = new ArrayList<>();
Throwable root = exception;
Set<Throwable> seen = new HashSet<>();
do
{
if(StringUtils.hasContent(root.getMessage()))
{
messages.add(root.getMessage());
}
else
{
messages.add(root.getClass().getSimpleName());
}
seen.add(root);
root = root.getCause();
}
while(root != null && !seen.contains(root));
return (StringUtils.join("; ", messages));
}
}

37
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ExceptionUtilsTest.java
View File

@ -26,6 +26,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
@ -88,6 +89,33 @@ class ExceptionUtilsTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testConcatenateMessagesFromChain()
{
assertNull(ExceptionUtils.concatenateMessagesFromChain(null));
assertEquals("QException", ExceptionUtils.concatenateMessagesFromChain(new QException((String) null)));
assertEquals("QException", ExceptionUtils.concatenateMessagesFromChain(new QException("")));
assertEquals("foo; bar", ExceptionUtils.concatenateMessagesFromChain(new QException("foo", new QException("bar"))));
assertEquals("foo; QException; bar", ExceptionUtils.concatenateMessagesFromChain(new QException("foo", new QException(null, new QException("bar")))));
MyException selfCaused = new MyException("selfCaused");
selfCaused.setCause(selfCaused);
assertEquals("selfCaused", ExceptionUtils.concatenateMessagesFromChain(selfCaused));
MyException cycle1 = new MyException("cycle1");
MyException cycle2 = new MyException("cycle2");
cycle1.setCause(cycle2);
cycle2.setCause(cycle1);
assertEquals("cycle1; cycle2", ExceptionUtils.concatenateMessagesFromChain(cycle1));
assertEquals("cycle2; cycle1", ExceptionUtils.concatenateMessagesFromChain(cycle2));
}
/*******************************************************************************
** Test exception class - lets you set the cause, easier to create a loop.
*******************************************************************************/
@ -97,6 +125,9 @@ class ExceptionUtilsTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
public MyException(String message)
{
super(message);
@ -104,6 +135,9 @@ class ExceptionUtilsTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
public MyException(Throwable cause)
{
super(cause);
@ -111,6 +145,9 @@ class ExceptionUtilsTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
public void setCause(Throwable cause)
{
myCause = cause;