QQQ-21 methods to run async process stuff

.circleci/config.yml

@@ -42,7 +42,7 @@ jobs:
     executor: java17
     steps:
       - run_maven:
-          maven_subcommand: test
+          maven_subcommand: verify
       - slack/notify:
           event: fail
 

checkstyle.xml

@@ -181,8 +181,8 @@
       </module>
       -->
       <module name="OverloadMethodsDeclarationOrder"/>
-      <module name="VariableDeclarationUsageDistance"/>
       <!--
+      <module name="VariableDeclarationUsageDistance"/>
       <module name="CustomImportOrder">
          <property name="sortImportsInGroupAlphabetically" value="true"/>
          <property name="separateLineBetweenGroups" value="true"/>

pom.xml

@@ -44,6 +44,8 @@
       <maven.compiler.target>17</maven.compiler.target>
       <maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>
       <maven.compiler.showWarnings>true</maven.compiler.showWarnings>
+      <coverage.haltOnFailure>true</coverage.haltOnFailure>
+      <coverage.instructionCoveredRatioMinimum>0.80</coverage.instructionCoveredRatioMinimum>
    </properties>
 
    <dependencies>
@@ -51,7 +53,7 @@
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-backend-core</artifactId>
-         <version>0.1.0-20220706.184937-2</version>
+         <version>0.1.0-20220708.152048-3</version>
       </dependency>
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
@@ -126,6 +128,9 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <version>3.0.0-M5</version>
+            <configuration>
+               <argLine>@{jaCoCoArgLine}</argLine>
+            </configuration>
          </plugin>
          <plugin>
             <groupId>org.apache.maven.plugins</groupId>
@@ -175,6 +180,84 @@
                <versionDigitToIncrement>1</versionDigitToIncrement> <!-- In general, we update the minor -->
             </configuration>
          </plugin>
+         <plugin>
+            <groupId>org.jacoco</groupId>
+            <artifactId>jacoco-maven-plugin</artifactId>
+            <version>0.8.8</version>
+            <executions>
+               <execution>
+                  <id>pre-unit-test</id>
+                  <goals>
+                     <goal>prepare-agent</goal>
+                  </goals>
+                  <configuration>
+                     <propertyName>jaCoCoArgLine</propertyName>
+                  </configuration>
+               </execution>
+               <execution>
+                  <id>unit-test-check</id>
+                  <goals>
+                     <goal>check</goal>
+                  </goals>
+                  <configuration>
+                     <!-- Gives us the ability to pass a parameter to not fail due to coverage E.g. -Dcoverage.haltOnFailure=false -->
+                     <haltOnFailure>${coverage.haltOnFailure}</haltOnFailure>
+                     <rules>
+                        <rule>
+                           <element>BUNDLE</element>
+                           <limits>
+                              <limit>
+                                 <counter>INSTRUCTION</counter>
+                                 <value>COVEREDRATIO</value>
+                                 <minimum>${coverage.instructionCoveredRatioMinimum}</minimum>
+                              </limit>
+                           </limits>
+                        </rule>
+                     </rules>
+                  </configuration>
+               </execution>
+               <execution>
+                  <id>post-unit-test</id>
+                  <phase>verify</phase>
+                  <goals>
+                     <goal>report</goal>
+                  </goals>
+               </execution>
+            </executions>
+         </plugin>
+         <plugin>
+            <artifactId>exec-maven-plugin</artifactId>
+            <groupId>org.codehaus.mojo</groupId>
+            <version>3.0.0</version>
+            <executions>
+               <execution>
+                  <id>test-coverage-summary</id>
+                  <phase>verify</phase>
+                  <goals>
+                     <goal>exec</goal>
+                  </goals>
+                  <configuration>
+                     <executable>sh</executable>
+                     <arguments>
+                        <argument>-c</argument>
+                        <argument>
+                           <![CDATA[
+echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
+xpath -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
+echo
+echo "Jacoco coverage summary report:"
+echo "   See also target/site/jacoco/index.html"
+echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
+echo "------------------------------------------------------------"
+paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
+rm /tmp/$$.headers /tmp/$$.values
+                           ]]>
+                        </argument>
+                     </arguments>
+                  </configuration>
+               </execution>
+            </executions>
+         </plugin>
       </plugins>
    </build>
 

src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java

@@ -29,10 +29,9 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.CompletableFuture;
+import java.util.Optional;
-import java.util.concurrent.CompletionException;
+import java.util.UUID;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import com.kingsrook.qqq.backend.core.actions.DeleteAction;
 import com.kingsrook.qqq.backend.core.actions.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.MetaDataAction;
@@ -41,11 +40,16 @@ import com.kingsrook.qqq.backend.core.actions.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.RunProcessAction;
 import com.kingsrook.qqq.backend.core.actions.TableMetaDataAction;
 import com.kingsrook.qqq.backend.core.actions.UpdateAction;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
+import com.kingsrook.qqq.backend.core.actions.async.JobGoingAsyncException;
 import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.exceptions.QValueException;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractQRequest;
 import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
 import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
@@ -57,6 +61,7 @@ import com.kingsrook.qqq.backend.core.model.actions.metadata.process.ProcessMeta
 import com.kingsrook.qqq.backend.core.model.actions.metadata.process.ProcessMetaDataResult;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.table.TableMetaDataRequest;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.table.TableMetaDataResult;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessRequest;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessResult;
 import com.kingsrook.qqq.backend.core.model.actions.query.QCriteriaOperator;
@@ -73,10 +78,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.modules.QAuthenticationModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.interfaces.QAuthenticationModuleInterface;
+import com.kingsrook.qqq.backend.core.state.StateType;
+import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import io.javalin.Javalin;
 import io.javalin.apibuilder.EndpointGroup;
 import io.javalin.http.Context;
@@ -105,7 +113,9 @@ public class QJavalinImplementation
 
    private static QInstance qInstance;
 
-   private static int PORT = 8001;
+   private static int DEFAULT_PORT = 8001;
+
+   private static int ASYNC_STEP_TIMEOUT_MILLIS = 3_000;
 
 
 
@@ -118,7 +128,7 @@ public class QJavalinImplementation
       // todo - parse args to look up metaData and prime instance
       // qInstance.addBackend(QMetaDataProvider.getQBackend());
 
-      new QJavalinImplementation(qInstance).startJavalinServer(PORT);
+      new QJavalinImplementation(qInstance).startJavalinServer(DEFAULT_PORT);
    }
 
 
@@ -158,6 +168,26 @@ public class QJavalinImplementation
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void setDefaultPort(int port)
+   {
+      QJavalinImplementation.DEFAULT_PORT = port;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void setAsyncStepTimeoutMillis(int asyncStepTimeoutMillis)
+   {
+      QJavalinImplementation.ASYNC_STEP_TIMEOUT_MILLIS = asyncStepTimeoutMillis;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -172,7 +202,7 @@ public class QJavalinImplementation
             {
                get("", QJavalinImplementation::tableMetaData);
             });
-            path("/process/:process", () ->
+            path("/process/:processName", () ->
             {
                get("", QJavalinImplementation::processMetaData);
             });
@@ -195,10 +225,16 @@ public class QJavalinImplementation
          });
          path("/processes", () ->
          {
-            path("/:process", () ->
+            path("/:processName", () ->
             {
                get("/init", QJavalinImplementation::processInit);
-               get("/step", QJavalinImplementation::processStep);
+               post("/init", QJavalinImplementation::processInit);
+
+               path("/:processUUID", () ->
+               {
+                  post("/step/:step", QJavalinImplementation::processStep);
+                  get("/status/:jobUUID", QJavalinImplementation::processStatus);
+               });
             });
          });
       });
@@ -481,7 +517,7 @@ public class QJavalinImplementation
       {
          ProcessMetaDataRequest processMetaDataRequest = new ProcessMetaDataRequest(qInstance);
          setupSession(context, processMetaDataRequest);
-         processMetaDataRequest.setProcessName(context.pathParam("process"));
+         processMetaDataRequest.setProcessName(context.pathParam("processName"));
          ProcessMetaDataAction processMetaDataAction = new ProcessMetaDataAction();
          ProcessMetaDataResult processMetaDataResult = processMetaDataAction.execute(processMetaDataRequest);
 
@@ -506,7 +542,7 @@ public class QJavalinImplementation
          if(userFacingException instanceof QNotFoundException)
          {
             context.status(HttpStatus.NOT_FOUND_404)
-               .result("{\"error\":\"" + e.getMessage() + "\"}");
+               .result("{\"error\":\"" + userFacingException.getMessage() + "\"}");
          }
          else
          {
@@ -527,15 +563,15 @@ public class QJavalinImplementation
 
    /*******************************************************************************
     ** Returns Integer if context has a valid int query parameter by the given name,
-    *  Returns null if no param (or empty value).
+    **  Returns null if no param (or empty value).
-    *  Throws NumberFormatException for malformed numbers.
+    **  Throws QValueException for malformed numbers.
     *******************************************************************************/
-   private static Integer integerQueryParam(Context context, String name) throws NumberFormatException
+   private static Integer integerQueryParam(Context context, String name) throws QValueException
    {
       String value = context.queryParam(name);
       if(StringUtils.hasContent(value))
       {
-         return (Integer.parseInt(value));
+         return (ValueUtils.getValueAsInteger(value));
       }
 
       return (null);
@@ -547,7 +583,7 @@ public class QJavalinImplementation
     ** Returns String if context has a valid query parameter by the given name,
     *  Returns null if no param (or empty value).
     *******************************************************************************/
-   private static String stringQueryParam(Context context, String name) throws NumberFormatException
+   private static String stringQueryParam(Context context, String name)
    {
       String value = context.queryParam(name);
       if(StringUtils.hasContent(value))
@@ -566,15 +602,138 @@ public class QJavalinImplementation
     *******************************************************************************/
    private static void processInit(Context context) throws QException
    {
+      doProcessInitOrStep(context, null, null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void doProcessInitOrStep(Context context, String processUUID, String startAfterStep) throws QModuleDispatchException
+   {
+      if(processUUID == null)
+      {
+         processUUID = UUID.randomUUID().toString();
+      }
+
       RunProcessRequest runProcessRequest = new RunProcessRequest(qInstance);
       setupSession(context, runProcessRequest);
-      runProcessRequest.setProcessName(context.pathParam("process"));
+      runProcessRequest.setProcessName(context.pathParam("processName"));
       runProcessRequest.setCallback(new QJavalinProcessCallback());
+      runProcessRequest.setBackendOnly(true);
+      runProcessRequest.setProcessUUID(processUUID);
+      runProcessRequest.setStartAfterStep(startAfterStep);
+      populateRunProcessRequestWithValuesFromContext(context, runProcessRequest);
 
-      /////////////////////////////////////////////////////////////////////////////////////
+      LOG.info(startAfterStep == null ? "Initiating process [" + runProcessRequest.getProcessName() + "] [" + processUUID + "]"
-      // take values from query-string params, and put them into the run process request //
+         : "Resuming process [" + runProcessRequest.getProcessName() + "] [" + processUUID + "] after step [" + startAfterStep + "]");
-      // todo - better from POST body, or with a "field-" type of prefix??               //
+
-      /////////////////////////////////////////////////////////////////////////////////////
+      Map<String, Object> resultForCaller = new HashMap<>();
+      resultForCaller.put("processUUID", processUUID);
+
+      try
+      {
+         ////////////////////////////////////////
+         // run the process as an async action //
+         ////////////////////////////////////////
+         Integer timeout = getTimeoutMillis(context);
+         RunProcessResult runProcessResult = new AsyncJobManager().startJob(timeout, TimeUnit.MILLISECONDS, (callback) ->
+         {
+            runProcessRequest.setAsyncJobCallback(callback);
+            return (new RunProcessAction().execute(runProcessRequest));
+         });
+
+         LOG.info("Process result error? " + runProcessResult.getException());
+         for(QFieldMetaData outputField : qInstance.getProcess(runProcessRequest.getProcessName()).getOutputFields())
+         {
+            LOG.info("Process result output value: " + outputField.getName() + ": " + runProcessResult.getValues().get(outputField.getName()));
+         }
+         serializeRunProcessResultForCaller(resultForCaller, runProcessResult);
+      }
+      catch(JobGoingAsyncException jgae)
+      {
+         resultForCaller.put("jobUUID", jgae.getJobUUID());
+      }
+      catch(Exception e)
+      {
+         //////////////////////////////////////////////////////////////////////////////
+         // our other actions in here would do: handleException(context, e);         //
+         // which would return a 500 to the client.                                  //
+         // but - other process-step actions, they always return a 200, just with an //
+         // optional error message - so - keep all of the processes consistent.      //
+         //////////////////////////////////////////////////////////////////////////////
+         serializeRunProcessExceptionForCaller(resultForCaller, e);
+      }
+
+      context.result(JsonUtils.toJson(resultForCaller));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static Integer getTimeoutMillis(Context context)
+   {
+      Integer timeout = integerQueryParam(context, "_qStepTimeoutMillis");
+      if(timeout == null)
+      {
+         timeout = ASYNC_STEP_TIMEOUT_MILLIS;
+      }
+      return timeout;
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a step finished synchronously or asynchronously, return its data
+    ** to the caller the same way.
+    *******************************************************************************/
+   private static void serializeRunProcessResultForCaller(Map<String, Object> resultForCaller, RunProcessResult runProcessResult)
+   {
+      if(runProcessResult.getException().isPresent())
+      {
+         ////////////////////////////////////////////////////////////////
+         // per code coverage, this path may never actually get hit... //
+         ////////////////////////////////////////////////////////////////
+         serializeRunProcessExceptionForCaller(resultForCaller, runProcessResult.getException().get());
+      }
+      resultForCaller.put("values", runProcessResult.getValues());
+      runProcessResult.getProcessState().getNextStepName().ifPresent(lastStep -> resultForCaller.put("nextStep", lastStep));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void serializeRunProcessExceptionForCaller(Map<String, Object> resultForCaller, Exception exception)
+   {
+      QUserFacingException userFacingException = ExceptionUtils.findClassInRootChain(exception, QUserFacingException.class);
+
+      if(userFacingException != null)
+      {
+         LOG.info("User-facing exception in process", userFacingException);
+         resultForCaller.put("error", userFacingException.getMessage()); // todo - put this somewhere else (make error an object w/ user-facing and/or other error?)
+      }
+      else
+      {
+         Throwable rootException = ExceptionUtils.getRootException(exception);
+         LOG.warn("Uncaught Exception in process", exception);
+         resultForCaller.put("error", "Original error message: " + rootException.getMessage());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** take values from query-string params, and put them into the run process request
+    ** todo - better from POST body, or with a "field-" type of prefix??
+    **
+    *******************************************************************************/
+   private static void populateRunProcessRequestWithValuesFromContext(Context context, RunProcessRequest runProcessRequest)
+   {
       for(Map.Entry<String, List<String>> queryParam : context.queryParamMap().entrySet())
       {
          String       fieldName = queryParam.getKey();
@@ -584,61 +743,74 @@ public class QJavalinImplementation
             runProcessRequest.addValue(fieldName, values.get(0));
          }
       }
-
-      try
-      {
-         ////////////////////////////////////////////////
-         // run the process                            //
-         // todo -  some "job id" to return to caller? //
-         ////////////////////////////////////////////////
-         CompletableFuture<RunProcessResult> future = CompletableFuture.supplyAsync(() ->
-         {
-            try
-            {
-               LOG.info("Running process [" + runProcessRequest.getProcessName() + "]");
-               RunProcessResult runProcessResult = new RunProcessAction().execute(runProcessRequest);
-               LOG.info("Process result error? " + runProcessResult.getError());
-               for(QFieldMetaData outputField : qInstance.getProcess(runProcessRequest.getProcessName()).getOutputFields())
-               {
-                  LOG.info("Process result output value: " + outputField.getName() + ": " + runProcessResult.getValues().get(outputField.getName()));
-               }
-               return (runProcessResult);
-            }
-            catch(Exception e)
-            {
-               LOG.error("Error running future for process", e);
-               throw (new CompletionException(e));
-            }
-         });
-
-         Map<String, Object> resultForCaller = new HashMap<>();
-         try
-         {
-            RunProcessResult runProcessResult = future.get(3, TimeUnit.SECONDS);
-            resultForCaller.put("error", runProcessResult.getError());
-            resultForCaller.put("values", runProcessResult.getValues());
-         }
-         catch(TimeoutException te)
-         {
-            resultForCaller.put("jobId", "Job is running asynchronously... job id available in a later version.");
-         }
-         context.result(JsonUtils.toJson(resultForCaller));
-      }
-      catch(Exception e)
-      {
-         handleException(context, e);
-      }
    }
 
 
 
    /*******************************************************************************
-    ** Run a step in a process (named in path param :process)
+    ** Run a step in a process (named in path param :processName)
     **
     *******************************************************************************/
-   private static void processStep(Context context)
+   private static void processStep(Context context) throws QModuleDispatchException
    {
-
+      String processUUID = context.pathParam("processUUID");
+      String lastStep    = context.pathParam("step");
+      doProcessInitOrStep(context, processUUID, lastStep);
    }
 
+
+
+   /*******************************************************************************
+    ** Get status for a currently running process (step)
+    *******************************************************************************/
+   private static void processStatus(Context context)
+   {
+      String processUUID = context.pathParam("processUUID");
+      String jobUUID     = context.pathParam("jobUUID");
+
+      LOG.info("Request for status of job " + jobUUID);
+      Optional<AsyncJobStatus> optionalJobStatus = new AsyncJobManager().getJobStatus(jobUUID);
+      if(optionalJobStatus.isEmpty())
+      {
+         handleException(context, new RuntimeException("Could not find status of process step job"));
+      }
+      else
+      {
+         Map<String, Object> resultForCaller = new HashMap<>();
+         AsyncJobStatus      jobStatus       = optionalJobStatus.get();
+
+         resultForCaller.put("jobStatus", jobStatus);
+         LOG.info("Job status is " + jobStatus.getState() + " for " + jobUUID);
+
+         if(jobStatus.getState().equals(AsyncJobState.COMPLETE))
+         {
+            ///////////////////////////////////////////////////////////////////////////////////////
+            // if the job is complete, get the process result from state provider, and return it //
+            // this output should look like it did if the job finished synchronously!!           //
+            ///////////////////////////////////////////////////////////////////////////////////////
+            Optional<ProcessState> processState = RunProcessAction.getStateProvider().get(ProcessState.class, new UUIDAndTypeStateKey(UUID.fromString(processUUID), StateType.PROCESS_STATUS));
+            if(processState.isPresent())
+            {
+               RunProcessResult runProcessResult = new RunProcessResult(processState.get());
+               serializeRunProcessResultForCaller(resultForCaller, runProcessResult);
+            }
+            else
+            {
+               handleException(context, new RuntimeException("Could not find process results"));
+            }
+         }
+         else if(jobStatus.getState().equals(AsyncJobState.ERROR))
+         {
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if the job had an error (e.g., a process step threw), "nicely" serialize its exception for the caller //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(jobStatus.getCaughtException() != null)
+            {
+               serializeRunProcessExceptionForCaller(resultForCaller, jobStatus.getCaughtException());
+            }
+         }
+
+         context.result(JsonUtils.toJson(resultForCaller));
+      }
+   }
 }

src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java

@@ -27,6 +27,7 @@ import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import kong.unirest.HttpResponse;
 import kong.unirest.Unirest;
@@ -37,6 +38,7 @@ import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -54,6 +56,9 @@ class QJavalinImplementationTest
    private static final int    PORT     = 6262;
    private static final String BASE_URL = "http://localhost:" + PORT;
 
+   private static final int MORE_THAN_TIMEOUT = 500;
+   private static final int LESS_THAN_TIMEOUT = 50;
+
 
 
    /*******************************************************************************
@@ -64,6 +69,7 @@ class QJavalinImplementationTest
    public static void beforeAll()
    {
       QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(TestUtils.defineInstance());
+      QJavalinImplementation.setAsyncStepTimeoutMillis(250);
       qJavalinImplementation.startJavalinServer(PORT);
    }
 
@@ -364,10 +370,7 @@ class QJavalinImplementationTest
    @Test
    public void test_dataDelete() throws Exception
    {
-      HttpResponse<String> response = Unirest.delete(BASE_URL + "/data/person/3")
+      HttpResponse<String> response = Unirest.delete(BASE_URL + "/data/person/3").asString();
-         .header("Content-Type", "application/json")
-         .asString();
-
       assertEquals(200, response.getStatus());
 
       JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
@@ -394,10 +397,7 @@ class QJavalinImplementationTest
    @Test
    public void test_processGreetInit()
    {
-      HttpResponse<String> response = Unirest.get(BASE_URL + "/processes/greet/init")
+      HttpResponse<String> response = Unirest.get(BASE_URL + "/processes/greet/init").asString();
-         .header("Content-Type", "application/json")
-         .asString();
-
       assertEquals(200, response.getStatus());
       JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
       assertNotNull(jsonObject);
@@ -413,14 +413,268 @@ class QJavalinImplementationTest
    @Test
    public void test_processGreetInitWithQueryValues()
    {
-      HttpResponse<String> response = Unirest.get(BASE_URL + "/processes/greet/init?greetingPrefix=Hey&greetingSuffix=Jude")
+      HttpResponse<String> response = Unirest.get(BASE_URL + "/processes/greet/init?greetingPrefix=Hey&greetingSuffix=Jude").asString();
-         .header("Content-Type", "application/json")
-         .asString();
-
       assertEquals(200, response.getStatus());
       JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
       assertNotNull(jsonObject);
       assertEquals("Hey X Jude", jsonObject.getJSONObject("values").getString("outputMessage"));
    }
 
+
+
+   /*******************************************************************************
+    ** test init'ing a process that goes async
+    **
+    *******************************************************************************/
+   @Test
+   public void test_processInitGoingAsync() throws InterruptedException
+   {
+      String               processBasePath = BASE_URL + "/processes/" + TestUtils.PROCESS_NAME_SIMPLE_SLEEP;
+      HttpResponse<String> response        = Unirest.get(processBasePath + "/init?" + TestUtils.SleeperStep.FIELD_SLEEP_MILLIS + "=" + MORE_THAN_TIMEOUT).asString();
+
+      JSONObject jsonObject  = assertProcessStepWentAsyncResponse(response);
+      String     processUUID = jsonObject.getString("processUUID");
+      String     jobUUID     = jsonObject.getString("jobUUID");
+      assertNotNull(processUUID, "Process UUID should not be null.");
+      assertNotNull(jobUUID, "Job UUID should not be null");
+
+      /////////////////////////////////////////////
+      // request job status before sleep is done //
+      /////////////////////////////////////////////
+      response = Unirest.get(processBasePath + "/" + processUUID + "/status/" + jobUUID).asString();
+      jsonObject = assertProcessStepRunningResponse(response);
+
+      ///////////////////////////////////
+      // sleep, to let that job finish //
+      ///////////////////////////////////
+      Thread.sleep(MORE_THAN_TIMEOUT);
+
+      ////////////////////////////////////////////////////////
+      // request job status again, get back results instead //
+      ////////////////////////////////////////////////////////
+      response = Unirest.get(processBasePath + "/" + processUUID + "/status/" + jobUUID).asString();
+      jsonObject = assertProcessStepCompleteResponse(response);
+   }
+
+
+
+   /*******************************************************************************
+    ** test init'ing a process that does NOT goes async
+    **
+    *******************************************************************************/
+   @Test
+   public void test_processInitNotGoingAsync()
+   {
+      HttpResponse<String> response = Unirest.post(BASE_URL + "/processes/" + TestUtils.PROCESS_NAME_SIMPLE_SLEEP + "/init?" + TestUtils.SleeperStep.FIELD_SLEEP_MILLIS + "=" + LESS_THAN_TIMEOUT)
+         .header("Content-Type", "application/json").asString();
+      assertProcessStepCompleteResponse(response);
+   }
+
+
+
+   /*******************************************************************************
+    ** test running a step a process that goes async
+    **
+    *******************************************************************************/
+   @Test
+   public void test_processStepGoingAsync() throws InterruptedException
+   {
+      /////////////////////////////////////////////
+      // first init the process, to get its UUID //
+      /////////////////////////////////////////////
+      String processBasePath = BASE_URL + "/processes/" + TestUtils.PROCESS_NAME_SLEEP_INTERACTIVE;
+      HttpResponse<String> response = Unirest.post(processBasePath + "/init?" + TestUtils.SleeperStep.FIELD_SLEEP_MILLIS + "=" + MORE_THAN_TIMEOUT)
+         .header("Content-Type", "application/json").asString();
+
+      JSONObject jsonObject  = assertProcessStepCompleteResponse(response);
+      String     processUUID = jsonObject.getString("processUUID");
+      String     nextStep    = jsonObject.getString("nextStep");
+      assertNotNull(processUUID, "Process UUID should not be null.");
+      assertNotNull(nextStep, "There should be a next step");
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // second, run the 'nextStep' (the backend step, that sleeps). run it with a long enough sleep so that it'll go async //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      response = Unirest.post(processBasePath + "/" + processUUID + "/step/" + nextStep)
+         .header("Content-Type", "application/json").asString();
+
+      jsonObject = assertProcessStepWentAsyncResponse(response);
+      String jobUUID = jsonObject.getString("jobUUID");
+
+      ///////////////////////////////////
+      // sleep, to let that job finish //
+      ///////////////////////////////////
+      Thread.sleep(MORE_THAN_TIMEOUT);
+
+      ///////////////////////////////
+      // third, request job status //
+      ///////////////////////////////
+      response = Unirest.get(processBasePath + "/" + processUUID + "/status/" + jobUUID).asString();
+
+      jsonObject = assertProcessStepCompleteResponse(response);
+      String nextStep2 = jsonObject.getString("nextStep");
+      assertNotNull(nextStep2, "There be one more next step");
+      assertNotEquals(nextStep, nextStep2, "The next step should be different this time.");
+   }
+
+
+
+   /*******************************************************************************
+    ** test running a step a process that does NOT goes async
+    **
+    *******************************************************************************/
+   @Test
+   public void test_processStepNotGoingAsync()
+   {
+      /////////////////////////////////////////////
+      // first init the process, to get its UUID //
+      /////////////////////////////////////////////
+      String processBasePath = BASE_URL + "/processes/" + TestUtils.PROCESS_NAME_SLEEP_INTERACTIVE;
+      HttpResponse<String> response = Unirest.post(processBasePath + "/init?" + TestUtils.SleeperStep.FIELD_SLEEP_MILLIS + "=" + LESS_THAN_TIMEOUT)
+         .header("Content-Type", "application/json").asString();
+
+      JSONObject jsonObject  = assertProcessStepCompleteResponse(response);
+      String     processUUID = jsonObject.getString("processUUID");
+      String     nextStep    = jsonObject.getString("nextStep");
+      assertNotNull(nextStep, "There should be a next step");
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // second, run the 'nextStep' (the backend step, that sleeps). run it with a short enough sleep so that it won't go async //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      response = Unirest.post(processBasePath + "/" + processUUID + "/step/" + nextStep)
+         .header("Content-Type", "application/json").asString();
+
+      jsonObject = assertProcessStepCompleteResponse(response);
+      String nextStep2 = jsonObject.getString("nextStep");
+      assertNotNull(nextStep2, "There be one more next step");
+      assertNotEquals(nextStep, nextStep2, "The next step should be different this time.");
+   }
+
+
+
+   /*******************************************************************************
+    ** test init'ing a process that goes async and then throws
+    **
+    *******************************************************************************/
+   @Test
+   public void test_processInitGoingAsyncThenThrowing() throws InterruptedException
+   {
+      String               processBasePath = BASE_URL + "/processes/" + TestUtils.PROCESS_NAME_SIMPLE_THROW;
+      HttpResponse<String> response        = Unirest.get(processBasePath + "/init?" + TestUtils.ThrowerStep.FIELD_SLEEP_MILLIS + "=" + MORE_THAN_TIMEOUT).asString();
+
+      JSONObject jsonObject  = assertProcessStepWentAsyncResponse(response);
+      String     processUUID = jsonObject.getString("processUUID");
+      String     jobUUID     = jsonObject.getString("jobUUID");
+
+      /////////////////////////////////////////////
+      // request job status before sleep is done //
+      /////////////////////////////////////////////
+      response = Unirest.get(processBasePath + "/" + processUUID + "/status/" + jobUUID).asString();
+      jsonObject = assertProcessStepRunningResponse(response);
+
+      ///////////////////////////////////
+      // sleep, to let that job finish //
+      ///////////////////////////////////
+      Thread.sleep(MORE_THAN_TIMEOUT);
+
+      /////////////////////////////////////////////////////////////
+      // request job status again, get back error status instead //
+      /////////////////////////////////////////////////////////////
+      response = Unirest.get(processBasePath + "/" + processUUID + "/status/" + jobUUID).asString();
+      jsonObject = assertProcessStepErrorResponse(response);
+   }
+
+
+
+   /*******************************************************************************
+    ** test init'ing a process that does NOT goes async, but throws.
+    **
+    *******************************************************************************/
+   @Test
+   public void test_processInitNotGoingAsyncButThrowing()
+   {
+      HttpResponse<String> response = Unirest.post(BASE_URL + "/processes/" + TestUtils.PROCESS_NAME_SIMPLE_THROW + "/init?" + TestUtils.ThrowerStep.FIELD_SLEEP_MILLIS + "=" + LESS_THAN_TIMEOUT)
+         .header("Content-Type", "application/json").asString();
+      assertProcessStepErrorResponse(response);
+   }
+
+
+
+   /*******************************************************************************
+    ** every time a process step (sync or async) has gone async, expect what the
+    ** response should look like
+    *******************************************************************************/
+   private JSONObject assertProcessStepWentAsyncResponse(HttpResponse<String> response)
+   {
+      assertEquals(200, response.getStatus());
+      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+
+      assertTrue(jsonObject.has("processUUID"), "Async-started response should have a processUUID");
+      assertTrue(jsonObject.has("jobUUID"), "Async-started response should have a jobUUID");
+
+      assertFalse(jsonObject.has("values"), "Async-started response should NOT have values");
+      assertFalse(jsonObject.has("error"), "Async-started response should NOT have error");
+
+      return (jsonObject);
+   }
+
+
+
+   /*******************************************************************************
+    ** every time a process step (sync or async) is still running, expect certain things
+    ** to be (and not to be) in the json response.
+    *******************************************************************************/
+   private JSONObject assertProcessStepRunningResponse(HttpResponse<String> response)
+   {
+      assertEquals(200, response.getStatus());
+      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+
+      assertTrue(jsonObject.has("jobStatus"), "Step Running response should have a jobStatus");
+
+      assertFalse(jsonObject.has("values"), "Step Running response should NOT have values");
+      assertFalse(jsonObject.has("error"), "Step Running response should NOT have error");
+
+      assertEquals(AsyncJobState.RUNNING.name(), jsonObject.getJSONObject("jobStatus").getString("state"));
+
+      return (jsonObject);
+   }
+
+
+
+   /*******************************************************************************
+    ** every time a process step (sync or async) completes, expect certain things
+    ** to be (and not to be) in the json response.
+    *******************************************************************************/
+   private JSONObject assertProcessStepCompleteResponse(HttpResponse<String> response)
+   {
+      assertEquals(200, response.getStatus());
+      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+
+      assertTrue(jsonObject.has("values"), "Step Complete response should have values");
+
+      assertFalse(jsonObject.has("jobUUID"), "Step Complete response should not have a jobUUID");
+      assertFalse(jsonObject.has("error"), "Step Complete response should not have an error");
+
+      return (jsonObject);
+   }
+
+
+
+   /*******************************************************************************
+    ** every time a process step (sync or async) has an error, expect certain things
+    ** to be (and not to be) in the json response.
+    *******************************************************************************/
+   private JSONObject assertProcessStepErrorResponse(HttpResponse<String> response)
+   {
+      assertEquals(200, response.getStatus());
+      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+
+      assertTrue(jsonObject.has("error"), "Step Error response should have an error");
+
+      assertFalse(jsonObject.has("jobUUID"), "Step Error response should not have a jobUUID");
+      assertFalse(jsonObject.has("values"), "Step Error response should not have values");
+
+      return (jsonObject);
+   }
+
 }

src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java

@@ -25,7 +25,12 @@ package com.kingsrook.qqq.backend.javalin;
 import java.io.InputStream;
 import java.sql.Connection;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QValueException;
+import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
 import com.kingsrook.qqq.backend.core.interfaces.mock.MockBackendStep;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
 import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
@@ -54,6 +59,15 @@ import static junit.framework.Assert.assertNotNull;
 public class TestUtils
 {
    public static final String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive";
+   public static final String PROCESS_NAME_SIMPLE_SLEEP             = "simpleSleep";
+   public static final String PROCESS_NAME_SIMPLE_THROW             = "simpleThrow";
+   public static final String PROCESS_NAME_SLEEP_INTERACTIVE        = "sleepInteractive";
+
+   public static final String STEP_NAME_SLEEPER = "sleeper";
+   public static final String STEP_NAME_THROWER = "thrower";
+
+   public static final String SCREEN_0 = "screen0";
+   public static final String SCREEN_1 = "screen1";
 
 
 
@@ -104,6 +118,9 @@ public class TestUtils
       qInstance.addTable(defineTablePerson());
       qInstance.addProcess(defineProcessGreetPeople());
       qInstance.addProcess(defineProcessGreetPeopleInteractive());
+      qInstance.addProcess(defineProcessSimpleSleep());
+      qInstance.addProcess(defineProcessScreenThenSleep());
+      qInstance.addProcess(defineProcessSimpleThrow());
       return (qInstance);
    }
 
@@ -175,7 +192,7 @@ public class TestUtils
             .withCode(new QCodeReference()
                .withName(MockBackendStep.class.getName())
                .withCodeType(QCodeType.JAVA)
-               .withCodeUsage(QCodeUsage.FUNCTION)) // todo - needed, or implied in this context?
+               .withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context?
             .withInputData(new QFunctionInputMetaData()
                .withRecordListMetaData(new QRecordListMetaData().withTableName("person"))
                .withFieldList(List.of(
@@ -213,7 +230,7 @@ public class TestUtils
             .withCode(new QCodeReference()
                .withName(MockBackendStep.class.getName())
                .withCodeType(QCodeType.JAVA)
-               .withCodeUsage(QCodeUsage.FUNCTION)) // todo - needed, or implied in this context?
+               .withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context?
             .withInputData(new QFunctionInputMetaData()
                .withRecordListMetaData(new QRecordListMetaData().withTableName("person"))
                .withFieldList(List.of(
@@ -234,4 +251,151 @@ public class TestUtils
          );
    }
 
+
+
+   /*******************************************************************************
+    ** Define a process with just one step that sleeps
+    *******************************************************************************/
+   private static QProcessMetaData defineProcessSimpleSleep()
+   {
+      return new QProcessMetaData()
+         .withName(PROCESS_NAME_SIMPLE_SLEEP)
+         .addStep(SleeperStep.getMetaData());
+   }
+
+
+
+   /*******************************************************************************
+    ** Define a process with a screen, then a sleep step
+    *******************************************************************************/
+   private static QProcessMetaData defineProcessScreenThenSleep()
+   {
+      return new QProcessMetaData()
+         .withName(PROCESS_NAME_SLEEP_INTERACTIVE)
+         .addStep(new QFrontendStepMetaData()
+            .withName(SCREEN_0)
+            .withFormField(new QFieldMetaData("outputMessage", QFieldType.STRING)))
+         .addStep(SleeperStep.getMetaData())
+         .addStep(new QFrontendStepMetaData()
+            .withName(SCREEN_1)
+            .withFormField(new QFieldMetaData("outputMessage", QFieldType.STRING)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Define a process with just one step that sleeps and then throws
+    *******************************************************************************/
+   private static QProcessMetaData defineProcessSimpleThrow()
+   {
+      return new QProcessMetaData()
+         .withName(PROCESS_NAME_SIMPLE_THROW)
+         .addStep(ThrowerStep.getMetaData());
+   }
+
+
+
+   /*******************************************************************************
+    ** Testing backend step - just sleeps however long you ask it to (or, throws if
+    ** you don't provide a number of seconds to sleep).
+    *******************************************************************************/
+   public static class SleeperStep implements BackendStep
+   {
+      public static final String FIELD_SLEEP_MILLIS = "sleepMillis";
+
+
+
+      /*******************************************************************************
+       ** Execute the backend step - using the request as input, and the result as output.
+       **
+       ******************************************************************************/
+      @Override
+      public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
+      {
+         try
+         {
+            Thread.sleep(runBackendStepRequest.getValueInteger(FIELD_SLEEP_MILLIS));
+         }
+         catch(InterruptedException e)
+         {
+            throw (new QException("Interrupted while sleeping..."));
+         }
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public static QBackendStepMetaData getMetaData()
+      {
+         return (new QBackendStepMetaData()
+            .withName(STEP_NAME_SLEEPER)
+            .withCode(new QCodeReference()
+               .withName(SleeperStep.class.getName())
+               .withCodeType(QCodeType.JAVA)
+               .withCodeUsage(QCodeUsage.BACKEND_STEP))
+            .withInputData(new QFunctionInputMetaData()
+               .addField(new QFieldMetaData(SleeperStep.FIELD_SLEEP_MILLIS, QFieldType.INTEGER))));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Testing backend step - just throws an exception after however long you ask it to sleep.
+    *******************************************************************************/
+   public static class ThrowerStep implements BackendStep
+   {
+      public static final String FIELD_SLEEP_MILLIS = "sleepMillis";
+
+
+
+      /*******************************************************************************
+       ** Execute the backend step - using the request as input, and the result as output.
+       **
+       ******************************************************************************/
+      @Override
+      public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
+      {
+         int sleepMillis;
+         try
+         {
+            sleepMillis = runBackendStepRequest.getValueInteger(FIELD_SLEEP_MILLIS);
+         }
+         catch(QValueException qve)
+         {
+            sleepMillis = 50;
+         }
+
+         try
+         {
+            Thread.sleep(sleepMillis);
+         }
+         catch(InterruptedException e)
+         {
+            throw (new QException("Interrupted while sleeping..."));
+         }
+
+         throw (new QException("I always throw."));
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public static QBackendStepMetaData getMetaData()
+      {
+         return (new QBackendStepMetaData()
+            .withName(STEP_NAME_THROWER)
+            .withCode(new QCodeReference()
+               .withName(ThrowerStep.class.getName())
+               .withCodeType(QCodeType.JAVA)
+               .withCodeUsage(QCodeUsage.BACKEND_STEP))
+            .withInputData(new QFunctionInputMetaData()
+               .addField(new QFieldMetaData(ThrowerStep.FIELD_SLEEP_MILLIS, QFieldType.INTEGER))));
+      }
+   }
+
 }