QQQ-21 checkpoint - async processes, refactoring of process state, exceptions

src/main/java/com/kingsrook/qqq/backend/core/actions/RunBackendStepAction.java

@@ -200,7 +200,7 @@ public class RunBackendStepAction
       catch(Exception e)
       {
          runBackendStepResult = new RunBackendStepResult();
-         runBackendStepResult.setError("Error running backend step code: " + e.getMessage());
+         runBackendStepResult.setException(e);
          LOG.info("Error running backend step code", e);
       }
 

src/main/java/com/kingsrook/qqq/backend/core/actions/RunProcessAction.java

@@ -22,19 +22,27 @@
 package com.kingsrook.qqq.backend.core.actions;
 
 
+import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.UUID;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessRequest;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessResult;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
 import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
-import com.kingsrook.qqq.backend.core.state.UUIDStateKey;
+import com.kingsrook.qqq.backend.core.state.StateType;
+import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
@@ -43,6 +51,9 @@ import com.kingsrook.qqq.backend.core.state.UUIDStateKey;
  *******************************************************************************/
 public class RunProcessAction
 {
+   private static final Logger LOG = LogManager.getLogger(RunProcessAction.class);
+
+
 
    /*******************************************************************************
     **
@@ -59,47 +70,70 @@ public class RunProcessAction
 
       RunProcessResult runProcessResult = new RunProcessResult();
 
-      UUIDStateKey         stateKey           = new UUIDStateKey();
+      //////////////////////////////////////////////////////////
-      RunBackendStepResult lastFunctionResult = null;
+      // generate a UUID for the process, if one wasn't given //
-
+      //////////////////////////////////////////////////////////
-      // todo - custom routing?
+      if(runProcessRequest.getProcessUUID() == null)
-      List<QStepMetaData> functionList = process.getStepList();
-      for(QStepMetaData function : functionList)
       {
-         RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(runProcessRequest.getInstance());
+         runProcessRequest.setProcessUUID(UUID.randomUUID().toString());
-
-         if(lastFunctionResult == null)
-         {
-            ///////////////////////////////////////////////////////////////////////////////////////////////////////
-            // for the first request, load state from the run process request to prime the run function request. //
-            ///////////////////////////////////////////////////////////////////////////////////////////////////////
-            primeFunction(runProcessRequest, runBackendStepRequest);
-         }
-         else
-         {
-            ////////////////////////////////////////////////////////////////////////////////////////
-            // for functions after the first one, load from state management to prime the request //
-            ////////////////////////////////////////////////////////////////////////////////////////
-            loadState(stateKey, runBackendStepRequest);
-         }
-
-         runBackendStepRequest.setProcessName(process.getName());
-         runBackendStepRequest.setStepName(function.getName());
-         runBackendStepRequest.setSession(runProcessRequest.getSession());
-         runBackendStepRequest.setCallback(runProcessRequest.getCallback());
-         lastFunctionResult = new RunBackendStepAction().execute(runBackendStepRequest);
-         if(lastFunctionResult.getError() != null)
-         {
-            runProcessResult.setError(lastFunctionResult.getError());
-            break;
-         }
-
-         storeState(stateKey, lastFunctionResult);
       }
+      runProcessResult.setProcessUUID(runProcessRequest.getProcessUUID());
 
-      if(lastFunctionResult != null)
+      UUIDAndTypeStateKey stateKey     = new UUIDAndTypeStateKey(UUID.fromString(runProcessRequest.getProcessUUID()), StateType.PROCESS_STATUS);
+      ProcessState        processState = primeProcessState(runProcessRequest, stateKey);
+
+      // todo - custom routing
+      List<QStepMetaData> stepList = getAvailableStepList(process, runProcessRequest);
+      try
       {
-         runProcessResult.seedFromLastFunctionResult(lastFunctionResult);
+         for(QStepMetaData step : stepList)
+         {
+            ////////////////////////////////////////////////////////////////////////////////////////////////
+            // if the caller requested to only run backend steps, then break if this isn't a backend step //
+            ////////////////////////////////////////////////////////////////////////////////////////////////
+            if(runProcessRequest.getBackendOnly())
+            {
+               if(!(step instanceof QBackendStepMetaData))
+               {
+                  LOG.info("Breaking process [" + process.getName() + "] at first non-backend step (as requested by caller): " + step.getName());
+                  processState.setNextStepName(step.getName());
+                  break;
+               }
+            }
+
+            /////////////////////////////////////
+            // run the step, based on its type //
+            /////////////////////////////////////
+            if(step instanceof QBackendStepMetaData backendStepMetaData)
+            {
+               runBackendStep(runProcessRequest, process, runProcessResult, stateKey, backendStepMetaData, processState);
+            }
+            else
+            {
+               throw (new QException("Unsure how to run a step of type: " + step.getClass().getName()));
+            }
+         }
+      }
+      catch(QException qe)
+      {
+         ////////////////////////////////////////////////////////////
+         // upon exception (e.g., one thrown by a step), throw it. //
+         ////////////////////////////////////////////////////////////
+         throw (qe);
+      }
+      catch(Exception e)
+      {
+         ////////////////////////////////////////////////////////////
+         // upon exception (e.g., one thrown by a step), throw it. //
+         ////////////////////////////////////////////////////////////
+         throw (new QException("Error running process", e));
+      }
+      finally
+      {
+         //////////////////////////////////////////////////////
+         // always put the final state in the process result //
+         //////////////////////////////////////////////////////
+         runProcessResult.setProcessState(processState);
       }
 
       return (runProcessResult);
@@ -107,11 +141,127 @@ public class RunProcessAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private ProcessState primeProcessState(RunProcessRequest runProcessRequest, UUIDAndTypeStateKey stateKey) throws QException
+   {
+      Optional<ProcessState> optionalProcessState = loadState(stateKey);
+      if(optionalProcessState.isEmpty())
+      {
+         if(runProcessRequest.getStartAfterStep() == null)
+         {
+            ///////////////////////////////////////////////////////////////////////////////////
+            // this is fine - it means its our first time running in the backend.            //
+            // Go ahead and store the state that we have (e.g., w/ initial records & values) //
+            ///////////////////////////////////////////////////////////////////////////////////
+            storeState(stateKey, runProcessRequest.getProcessState());
+            optionalProcessState = Optional.of(runProcessRequest.getProcessState());
+         }
+         else
+         {
+            ////////////////////////////////////////////////////////////////////////////////////////
+            // if this isn't the first step, but there's no state, then that's a problem, so fail //
+            ////////////////////////////////////////////////////////////////////////////////////////
+            throw (new QException("Could not find state for process [" + runProcessRequest.getProcessName() + "] [" + stateKey.getUuid() + "] in state provider."));
+         }
+      }
+      else
+      {
+         //////////////////////////////////////////////////////////////////////////////////////////////////////
+         // capture any values that the caller may have supplied in the request, before restoring from state //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////
+         Map<String, Serializable> valuesFromCaller = runProcessRequest.getValues();
+
+         ///////////////////////////////////////////////////
+         // if there is a previously stored state, use it //
+         ///////////////////////////////////////////////////
+         runProcessRequest.seedFromProcessState(optionalProcessState.get());
+
+         ///////////////////////////////////////////////////////////////////////////
+         // if there were values from the caller, put those (back) in the request //
+         ///////////////////////////////////////////////////////////////////////////
+         if(valuesFromCaller != null)
+         {
+            for(Map.Entry<String, Serializable> entry : valuesFromCaller.entrySet())
+            {
+               runProcessRequest.addValue(entry.getKey(), entry.getValue());
+            }
+         }
+      }
+
+      ProcessState processState = optionalProcessState.get();
+      processState.clearNextStepName();
+      return processState;
+   }
+
+
+
+   /*******************************************************************************
+    ** return true if 'ok', false if error (and time to break loop)
+    *******************************************************************************/
+   private void runBackendStep(RunProcessRequest runProcessRequest, QProcessMetaData process, RunProcessResult runProcessResult, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, ProcessState processState) throws Exception
+   {
+      RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(runProcessRequest.getInstance(), processState);
+      runBackendStepRequest.setProcessName(process.getName());
+      runBackendStepRequest.setStepName(backendStep.getName());
+      runBackendStepRequest.setSession(runProcessRequest.getSession());
+      runBackendStepRequest.setCallback(runProcessRequest.getCallback());
+      RunBackendStepResult lastFunctionResult = new RunBackendStepAction().execute(runBackendStepRequest);
+      storeState(stateKey, lastFunctionResult.getProcessState());
+
+      if(lastFunctionResult.getException() != null)
+      {
+         runProcessResult.setException(lastFunctionResult.getException());
+         throw (lastFunctionResult.getException());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Get the list of steps which are eligible to run.
+    *******************************************************************************/
+   private List<QStepMetaData> getAvailableStepList(QProcessMetaData process, RunProcessRequest runProcessRequest)
+   {
+      if(runProcessRequest.getStartAfterStep() == null)
+      {
+         /////////////////////////////////////////////////////////////////////////////
+         // if the caller did not supply a 'startAfterStep', then use the full list //
+         /////////////////////////////////////////////////////////////////////////////
+         return (process.getStepList());
+      }
+      else
+      {
+         ////////////////////////////////////////////////////////////////////////////////
+         // else, loop until the startAfterStep is found, and return the ones after it //
+         ////////////////////////////////////////////////////////////////////////////////
+         boolean             foundStartAfterStep = false;
+         List<QStepMetaData> rs                  = new ArrayList<>();
+
+         for(QStepMetaData step : process.getStepList())
+         {
+            if(foundStartAfterStep)
+            {
+               rs.add(step);
+            }
+
+            if(step.getName().equals(runProcessRequest.getStartAfterStep()))
+            {
+               foundStartAfterStep = true;
+            }
+         }
+         return (rs);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Load an instance of the appropriate state provider
     **
     *******************************************************************************/
-   private StateProviderInterface getStateProvider()
+   public static StateProviderInterface getStateProvider()
    {
       // TODO - read this from somewhere in meta data eh?
       return InMemoryStateProvider.getInstance();
@@ -126,33 +276,20 @@ public class RunProcessAction
     ** Store the process state from a function result to the state provider
     **
     *******************************************************************************/
-   private void storeState(UUIDStateKey stateKey, RunBackendStepResult runBackendStepResult)
+   private void storeState(UUIDAndTypeStateKey stateKey, ProcessState processState)
    {
-      getStateProvider().put(stateKey, runBackendStepResult.getProcessState());
+      getStateProvider().put(stateKey, processState);
    }
 
 
 
    /*******************************************************************************
-    ** Copy data (the state) down from the run-process request, down into the run-
+    ** Load the process state.
-    ** function request.
-    *******************************************************************************/
-   private void primeFunction(RunProcessRequest runProcessRequest, RunBackendStepRequest runBackendStepRequest)
-   {
-      runBackendStepRequest.seedFromRunProcessRequest(runProcessRequest);
-   }
-
-
-
-   /*******************************************************************************
-    ** Load the process state into a function request from the state provider
     **
     *******************************************************************************/
-   private void loadState(UUIDStateKey stateKey, RunBackendStepRequest runBackendStepRequest) throws QException
+   private Optional<ProcessState> loadState(UUIDAndTypeStateKey stateKey)
    {
-      Optional<ProcessState> processState = getStateProvider().get(ProcessState.class, stateKey);
+      return (getStateProvider().get(ProcessState.class, stateKey));
-      runBackendStepRequest.seedFromProcessState(processState
-         .orElseThrow(() -> new QException("Could not find process state in state provider.")));
    }
 
 }

src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJob.java

@@ -0,0 +1,36 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.async;
+
+
+/*******************************************************************************
+ ** Interface to be implemented (as lambdas), for working with AsyncJobManager.
+ *******************************************************************************/
+@FunctionalInterface
+public interface AsyncJob<T>
+{
+   /*******************************************************************************
+    ** Run the job, taking a callback object (where you can communicate your status
+    ** back), returning a result when you're done..
+    *******************************************************************************/
+   T run(AsyncJobCallback callback) throws Exception;
+}

src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobCallback.java

@@ -19,85 +19,80 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.model.metadata.processes;
+package com.kingsrook.qqq.backend.core.actions.async;
+
+
+import java.util.UUID;
+import com.kingsrook.qqq.backend.core.state.StateType;
+import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
 
 
 /*******************************************************************************
- ** Meta-Data to define the Output View for a QQQ Function
+ ** Argument passed to an AsyncJob when it runs, which can be used to communicate
+ ** data back out of the job.
  **
+ ** TODO - future - allow cancellation to be indicated here?
  *******************************************************************************/
-public class QOutputView
+public class AsyncJobCallback
 {
-   private String messageField;
+   private UUID           jobUUID;
-   private QRecordListView recordListView;
+   private AsyncJobStatus asyncJobStatus;
 
 
 
    /*******************************************************************************
-    ** Getter for message
     **
     *******************************************************************************/
-   public String getMessageField()
+   public AsyncJobCallback(UUID jobUUID, AsyncJobStatus asyncJobStatus)
    {
-      return messageField;
+      this.jobUUID = jobUUID;
+      this.asyncJobStatus = asyncJobStatus;
    }
 
 
 
    /*******************************************************************************
-    ** Setter for message
     **
     *******************************************************************************/
-   public void setMessage(String message)
+   public void updateStatus(String message)
    {
-      this.messageField = message;
+      this.asyncJobStatus.setMessage(message);
+      storeUpdatedStatus();
    }
 
 
 
    /*******************************************************************************
-    ** Setter for message
     **
     *******************************************************************************/
-   public QOutputView withMessageField(String messageField)
+   public void updateStatus(int current, int total)
    {
-      this.messageField = messageField;
+      this.asyncJobStatus.setCurrent(current);
-      return(this);
+      this.asyncJobStatus.setTotal(total);
+      storeUpdatedStatus();
    }
 
 
 
    /*******************************************************************************
-    ** Getter for recordListView
     **
     *******************************************************************************/
-   public QRecordListView getRecordListView()
+   public void updateStatus(String message, int current, int total)
    {
-      return recordListView;
+      this.asyncJobStatus.setMessage(message);
+      this.asyncJobStatus.setCurrent(current);
+      this.asyncJobStatus.setTotal(total);
+      storeUpdatedStatus();
    }
 
 
 
    /*******************************************************************************
-    ** Setter for recordListView
     **
     *******************************************************************************/
-   public void setRecordListView(QRecordListView recordListView)
+   private void storeUpdatedStatus()
    {
-      this.recordListView = recordListView;
+      AsyncJobManager.getStateProvider().put(new UUIDAndTypeStateKey(jobUUID, StateType.ASYNC_JOB_STATUS), asyncJobStatus);
    }
 
-
-
-   /*******************************************************************************
-    ** Setter for recordListView
-    **
-    *******************************************************************************/
-   public QOutputView withRecordListView(QRecordListView recordListView)
-   {
-      this.recordListView = recordListView;
-      return(this);
-   }
-
-
 }

src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java

@@ -0,0 +1,125 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.async;
+
+
+import java.io.Serializable;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
+import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
+import com.kingsrook.qqq.backend.core.state.StateType;
+import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+
+/*******************************************************************************
+ ** Class to manage running asynchronous actions, and working with their statuses.
+ *******************************************************************************/
+public class AsyncJobManager
+{
+   private static final Logger LOG = LogManager.getLogger(AsyncJobManager.class);
+
+
+
+   /*******************************************************************************
+    ** Run a job - if it finishes within the specified timeout, get its results,
+    ** else, get back an exception with the job id.
+    *******************************************************************************/
+   public <T extends Serializable> T startJob(long timeout, TimeUnit timeUnit, AsyncJob<T> asyncJob) throws JobGoingAsyncException, QException
+   {
+      UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.ASYNC_JOB_STATUS);
+      AsyncJobStatus      asyncJobStatus      = new AsyncJobStatus();
+      asyncJobStatus.setState(AsyncJobState.RUNNING);
+      getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
+
+      try
+      {
+         CompletableFuture<T> future = CompletableFuture.supplyAsync(() ->
+         {
+            try
+            {
+               LOG.info("Starting job " + uuidAndTypeStateKey.getUuid());
+               T result = asyncJob.run(new AsyncJobCallback(uuidAndTypeStateKey.getUuid(), asyncJobStatus));
+               asyncJobStatus.setState(AsyncJobState.COMPLETE);
+               getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
+               LOG.info("Completed job " + uuidAndTypeStateKey.getUuid());
+               return (result);
+            }
+            catch(Exception e)
+            {
+               asyncJobStatus.setState(AsyncJobState.ERROR);
+               asyncJobStatus.setCaughtException(e);
+               getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
+               throw (new CompletionException(e));
+            }
+         });
+
+         T result = future.get(timeout, timeUnit);
+         return (result);
+      }
+      catch(InterruptedException | ExecutionException e)
+      {
+         throw (new QException("Error running job", e));
+      }
+      catch(TimeoutException e)
+      {
+         LOG.info("Job going async " + uuidAndTypeStateKey.getUuid());
+         throw (new JobGoingAsyncException(uuidAndTypeStateKey.getUuid().toString()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Get the status of the job identified by the given UUID.
+    **
+    *******************************************************************************/
+   public Optional<AsyncJobStatus> getJobStatus(String uuid)
+   {
+      UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(UUID.fromString(uuid), StateType.ASYNC_JOB_STATUS);
+      return (getStateProvider().get(AsyncJobStatus.class, uuidAndTypeStateKey));
+   }
+
+
+
+   /*******************************************************************************
+    ** Load an instance of the appropriate state provider
+    **
+    *******************************************************************************/
+   protected static StateProviderInterface getStateProvider()
+   {
+      // TODO - read this from somewhere in meta data eh?
+      return InMemoryStateProvider.getInstance();
+
+      // todo - by using JSON serialization internally, this makes stupidly large payloads and crashes things.
+      // return TempFileStateProvider.getInstance();
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobState.java

@@ -0,0 +1,33 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.async;
+
+
+/*******************************************************************************
+ ** Possible states for an async job's "running"-ness.
+ *******************************************************************************/
+public enum AsyncJobState
+{
+   RUNNING,
+   COMPLETE,
+   ERROR
+}

src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java

@@ -0,0 +1,149 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.async;
+
+
+import java.io.Serializable;
+
+
+/*******************************************************************************
+ ** Object to track current status of an async job - e.g., its state, and some
+ ** messages from the backend like "x of y"
+ *******************************************************************************/
+public class AsyncJobStatus implements Serializable
+{
+   private AsyncJobState state;
+   private String        message;
+   private Integer       current;
+   private Integer       total;
+   private Exception     caughtException;
+
+
+
+   /*******************************************************************************
+    ** Getter for state
+    **
+    *******************************************************************************/
+   public AsyncJobState getState()
+   {
+      return state;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for state
+    **
+    *******************************************************************************/
+   public void setState(AsyncJobState state)
+   {
+      this.state = state;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for message
+    **
+    *******************************************************************************/
+   public String getMessage()
+   {
+      return message;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for message
+    **
+    *******************************************************************************/
+   public void setMessage(String message)
+   {
+      this.message = message;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for current
+    **
+    *******************************************************************************/
+   public Integer getCurrent()
+   {
+      return current;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for current
+    **
+    *******************************************************************************/
+   public void setCurrent(Integer current)
+   {
+      this.current = current;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for total
+    **
+    *******************************************************************************/
+   public Integer getTotal()
+   {
+      return total;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for total
+    **
+    *******************************************************************************/
+   public void setTotal(Integer total)
+   {
+      this.total = total;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for caughtException
+    **
+    *******************************************************************************/
+   public Exception getCaughtException()
+   {
+      return caughtException;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for caughtException
+    **
+    *******************************************************************************/
+   public void setCaughtException(Exception caughtException)
+   {
+      this.caughtException = caughtException;
+   }
+}

src/main/java/com/kingsrook/qqq/backend/core/actions/async/JobGoingAsyncException.java

@@ -0,0 +1,53 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.async;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class JobGoingAsyncException extends Exception
+{
+   private String jobUUID;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public JobGoingAsyncException(String jobUUID)
+   {
+      this.jobUUID = jobUUID;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for jobUUID
+    **
+    *******************************************************************************/
+   public String getJobUUID()
+   {
+      return jobUUID;
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/core/exceptions/QValueException.java

@@ -0,0 +1,54 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.exceptions;
+
+
+/*******************************************************************************
+ ** Exception for when there's a problem with a value (like a string that you need
+ ** to be an integer, but it isn't).
+ **
+ *******************************************************************************/
+public class QValueException extends RuntimeException
+{
+
+
+   /*******************************************************************************
+    ** Constructor of message
+    **
+    *******************************************************************************/
+   public QValueException(String message)
+   {
+      super(message);
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor of message & cause
+    **
+    *******************************************************************************/
+   public QValueException(String message, Throwable cause)
+   {
+      super(message, cause);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -27,6 +27,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -111,15 +112,20 @@ public class QInstanceEnricher
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void enrich(QStepMetaData function)
+   private void enrich(QStepMetaData step)
    {
-      if(!StringUtils.hasContent(function.getLabel()))
+      if(!StringUtils.hasContent(step.getLabel()))
       {
-         function.setLabel(nameToLabel(function.getName()));
+         step.setLabel(nameToLabel(step.getName()));
       }
 
-      function.getInputFields().forEach(this::enrich);
+      step.getInputFields().forEach(this::enrich);
-      function.getOutputFields().forEach(this::enrich);
+      step.getOutputFields().forEach(this::enrich);
+
+      if (step instanceof QFrontendStepMetaData)
+      {
+         ((QFrontendStepMetaData)step).getFormFields().forEach(this::enrich);
+      }
    }
 
 

src/main/java/com/kingsrook/qqq/backend/core/interfaces/mock/MockBackendStep.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.interfaces.mock;
 
 
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
@@ -38,7 +39,7 @@ public class MockBackendStep implements BackendStep
    public static final String FIELD_GREETING_SUFFIX = "greetingSuffix";
 
    @Override
-   public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult)
+   public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
    {
       runBackendStepResult.getRecords().forEach(r -> r.setValue("mockValue", "Ha ha!"));
 
@@ -49,5 +50,10 @@ public class MockBackendStep implements BackendStep
       // mock the "greet" process... //
       /////////////////////////////////
       runBackendStepResult.addValue("outputMessage", runBackendStepRequest.getValueString(FIELD_GREETING_PREFIX) + " X " + runBackendStepRequest.getValueString(FIELD_GREETING_SUFFIX));
+
+      if("there".equalsIgnoreCase(runBackendStepRequest.getValueString(FIELD_GREETING_SUFFIX)))
+      {
+         throw (new QException("You said Hello There, didn't you..."));
+      }
    }
 }

src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java

@@ -27,6 +27,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
@@ -35,8 +36,9 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  *******************************************************************************/
 public class ProcessState implements Serializable
 {
-   private List<QRecord> records = new ArrayList<>();
+   private List<QRecord>             records      = new ArrayList<>();
-   private Map<String, Serializable> values = new HashMap<>();
+   private Map<String, Serializable> values       = new HashMap<>();
+   private Optional<String>          nextStepName = Optional.empty();
 
 
 
@@ -81,4 +83,38 @@ public class ProcessState implements Serializable
    {
       this.values = values;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for nextStepName
+    **
+    *******************************************************************************/
+   public Optional<String> getNextStepName()
+   {
+      return nextStepName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for nextStepName
+    **
+    *******************************************************************************/
+   public void setNextStepName(String nextStepName)
+   {
+      this.nextStepName = Optional.of(nextStepName);
+   }
+
+
+
+   /*******************************************************************************
+    ** clear out the value of nextStepName (set the Optional to empty)
+    **
+    *******************************************************************************/
+   public void clearNextStepName()
+   {
+      this.nextStepName = Optional.empty();
+   }
+
 }

src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepRequest.java

@@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractQRequest;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
@@ -67,27 +68,16 @@ public class RunBackendStepRequest extends AbstractQRequest
 
 
    /*******************************************************************************
-    ** e.g., for steps after the first step in a process, seed the data in a run
-    ** function request from a process state.
     **
     *******************************************************************************/
-   public void seedFromProcessState(ProcessState processState)
+   public RunBackendStepRequest(QInstance instance, ProcessState processState)
    {
+      super(instance);
       this.processState = processState;
    }
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public void seedFromRunProcessRequest(RunProcessRequest runProcessRequest)
-   {
-      this.processState = runProcessRequest.getProcessState();
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -308,7 +298,7 @@ public class RunBackendStepRequest extends AbstractQRequest
     *******************************************************************************/
    public Integer getValueInteger(String fieldName)
    {
-      return ((Integer) getValue(fieldName));
+      return (ValueUtils.getValueAsInteger(getValue(fieldName)));
    }
 
 

src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepResult.java

@@ -36,7 +36,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 public class RunBackendStepResult extends AbstractQResult
 {
    private ProcessState processState;
-   private String error;
+   private Exception    exception;
 
 
 
@@ -46,7 +46,7 @@ public class RunBackendStepResult extends AbstractQResult
    @Override
    public String toString()
    {
-      return "RunBackendStepResult{error='" + error
+      return "RunBackendStepResult{exception?='" + exception.getMessage()
          + ",records.size()=" + (processState == null ? null : processState.getRecords().size())
          + ",values=" + (processState == null ? null : processState.getValues())
          + "}";
@@ -155,28 +155,6 @@ public class RunBackendStepResult extends AbstractQResult
 
 
 
-   /*******************************************************************************
-    ** Getter for error
-    **
-    *******************************************************************************/
-   public String getError()
-   {
-      return error;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for error
-    **
-    *******************************************************************************/
-   public void setError(String error)
-   {
-      this.error = error;
-   }
-
-
-
    /*******************************************************************************
     ** Accessor for processState
     **
@@ -185,4 +163,24 @@ public class RunBackendStepResult extends AbstractQResult
    {
       return processState;
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setException(Exception exception)
+   {
+      this.exception = exception;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Exception getException()
+   {
+      return exception;
+   }
 }

src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessRequest.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.processes;
 import java.io.Serializable;
 import java.util.List;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback;
 import com.kingsrook.qqq.backend.core.callbacks.QProcessCallback;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractQRequest;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -38,9 +39,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
  *******************************************************************************/
 public class RunProcessRequest extends AbstractQRequest
 {
-   private String processName;
+   private String           processName;
    private QProcessCallback callback;
-   private ProcessState processState;
+   private ProcessState     processState;
+   private boolean          backendOnly = false;
+   private String           startAfterStep;
+   private String           processUUID;
+   private AsyncJobCallback asyncJobCallback;
 
 
 
@@ -65,6 +70,18 @@ public class RunProcessRequest extends AbstractQRequest
 
 
 
+   /*******************************************************************************
+    ** e.g., for steps after the first step in a process, seed the data in a run
+    ** function request from a process state.
+    **
+    *******************************************************************************/
+   public void seedFromProcessState(ProcessState processState)
+   {
+      this.processState = processState;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -255,14 +272,93 @@ public class RunProcessRequest extends AbstractQRequest
    }
 
 
+
    /*******************************************************************************
-    ** Accessor for processState - protected, because we generally want to access
+    ** Accessor for processState
-    ** its members through wrapper methods, we think
     **
     *******************************************************************************/
-   protected ProcessState getProcessState()
+   public ProcessState getProcessState()
    {
       return processState;
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setBackendOnly(boolean backendOnly)
+   {
+      this.backendOnly = backendOnly;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public boolean getBackendOnly()
+   {
+      return backendOnly;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setStartAfterStep(String startAfterStep)
+   {
+      this.startAfterStep = startAfterStep;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String getStartAfterStep()
+   {
+      return startAfterStep;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setProcessUUID(String processUUID)
+   {
+      this.processUUID = processUUID;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String getProcessUUID()
+   {
+      return processUUID;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setAsyncJobCallback(AsyncJobCallback asyncJobCallback)
+   {
+      this.asyncJobCallback = asyncJobCallback;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public AsyncJobCallback getAsyncJobCallback()
+   {
+      return asyncJobCallback;
+   }
 }

src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessResult.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.processes;
 import java.io.Serializable;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractQResult;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
@@ -33,24 +34,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** Result data container for the RunProcess action
  **
  *******************************************************************************/
-public class RunProcessResult extends AbstractQResult
+public class RunProcessResult extends AbstractQResult implements Serializable
 {
-   private ProcessState processState;
+   private ProcessState        processState;
-   private String       error;
+   private String              processUUID;
-
+   private Optional<Exception> exception = Optional.empty();
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public String toString()
-   {
-      return "RunProcessResult{error='" + error
-         + ",records.size()=" + (processState == null ? null : processState.getRecords().size())
-         + ",values=" + (processState == null ? null : processState.getValues())
-         + "}";
-   }
 
 
 
@@ -65,13 +53,26 @@ public class RunProcessResult extends AbstractQResult
 
 
    /*******************************************************************************
-    ** e.g., populate the process state (records, values) in this result object from
-    ** the final function result
     **
     *******************************************************************************/
-   public void seedFromLastFunctionResult(RunBackendStepResult runBackendStepResult)
+   public RunProcessResult(ProcessState processState)
    {
-      this.processState = runBackendStepResult.getProcessState();
+      this.processState = processState;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String toString()
+   {
+      return "RunProcessResult{uuid='" + processUUID
+         + ",exception?='" + (exception.isPresent() ? exception.get().getMessage() : "null")
+         + ",records.size()=" + (processState == null ? null : processState.getRecords().size())
+         + ",values=" + (processState == null ? null : processState.getValues())
+         + "}";
    }
 
 
@@ -157,22 +158,64 @@ public class RunProcessResult extends AbstractQResult
 
 
    /*******************************************************************************
-    ** Getter for error
+    ** Getter for processUUID
     **
     *******************************************************************************/
-   public String getError()
+   public String getProcessUUID()
    {
-      return error;
+      return processUUID;
    }
 
 
 
    /*******************************************************************************
-    ** Setter for error
+    ** Setter for processUUID
     **
     *******************************************************************************/
-   public void setError(String error)
+   public void setProcessUUID(String processUUID)
    {
-      this.error = error;
+      this.processUUID = processUUID;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for processState
+    **
+    *******************************************************************************/
+   public ProcessState getProcessState()
+   {
+      return processState;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for processState
+    **
+    *******************************************************************************/
+   public void setProcessState(ProcessState processState)
+   {
+      this.processState = processState;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setException(Exception exception)
+   {
+      this.exception = Optional.of(exception);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Optional<Exception> getException()
+   {
+      return exception;
    }
 }

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QCodeUsage.java

@@ -28,6 +28,6 @@ package com.kingsrook.qqq.backend.core.model.metadata;
  *******************************************************************************/
 public enum QCodeUsage
 {
-   FUNCTION, // a step in a process
+   BACKEND_STEP, // a backend-step in a process
    CUSTOMIZER // a function to customize part of a QQQ table's behavior
 }

src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java

@@ -60,7 +60,7 @@ public class BasicETLProcess
          .withCode(new QCodeReference()
             .withName(BasicETLExtractFunction.class.getName())
             .withCodeType(QCodeType.JAVA)
-            .withCodeUsage(QCodeUsage.FUNCTION))
+            .withCodeUsage(QCodeUsage.BACKEND_STEP))
          .withInputData(new QFunctionInputMetaData()
             .addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)));
 
@@ -69,7 +69,7 @@ public class BasicETLProcess
          .withCode(new QCodeReference()
             .withName(BasicETLTransformFunction.class.getName())
             .withCodeType(QCodeType.JAVA)
-            .withCodeUsage(QCodeUsage.FUNCTION))
+            .withCodeUsage(QCodeUsage.BACKEND_STEP))
          .withInputData(new QFunctionInputMetaData()
             .addField(new QFieldMetaData(FIELD_MAPPING_JSON, QFieldType.STRING))
             .addField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)));
@@ -79,7 +79,7 @@ public class BasicETLProcess
          .withCode(new QCodeReference()
             .withName(BasicETLLoadFunction.class.getName())
             .withCodeType(QCodeType.JAVA)
-            .withCodeUsage(QCodeUsage.FUNCTION))
+            .withCodeUsage(QCodeUsage.BACKEND_STEP))
          .withInputData(new QFunctionInputMetaData()
             .addField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)))
          .withOutputMetaData(new QFunctionOutputMetaData()

src/main/java/com/kingsrook/qqq/backend/core/state/StateType.java

@@ -0,0 +1,32 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.state;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public enum StateType
+{
+   PROCESS_STATUS,
+   ASYNC_JOB_STATUS
+}

src/main/java/com/kingsrook/qqq/backend/core/state/UUIDAndTypeStateKey.java

@@ -29,9 +29,10 @@ import java.util.UUID;
 /*******************************************************************************
  **
  *******************************************************************************/
-public class UUIDStateKey extends AbstractStateKey
+public class UUIDAndTypeStateKey extends AbstractStateKey
 {
-   private final UUID uuid;
+   private final UUID      uuid;
+   private final StateType stateType;
 
 
 
@@ -39,20 +40,21 @@ public class UUIDStateKey extends AbstractStateKey
     ** Default constructor - assigns a random UUID.
     **
     *******************************************************************************/
-   public UUIDStateKey()
+   public UUIDAndTypeStateKey(StateType stateType)
    {
-      uuid = UUID.randomUUID();
+      this(UUID.randomUUID(), stateType);
    }
 
 
 
    /*******************************************************************************
-    ** Constructor that lets you supply a UUID.
+    ** Constructor where user can supply the UUID.
     **
     *******************************************************************************/
-   public UUIDStateKey(UUID uuid)
+   public UUIDAndTypeStateKey(UUID uuid, StateType stateType)
    {
       this.uuid = uuid;
+      this.stateType = stateType;
    }
 
 
@@ -68,6 +70,17 @@ public class UUIDStateKey extends AbstractStateKey
 
 
 
+   /*******************************************************************************
+    ** Getter for stateType
+    **
+    *******************************************************************************/
+   public StateType getStateType()
+   {
+      return stateType;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -78,14 +91,12 @@ public class UUIDStateKey extends AbstractStateKey
       {
          return true;
       }
-
       if(o == null || getClass() != o.getClass())
       {
          return false;
       }
-
+      UUIDAndTypeStateKey that = (UUIDAndTypeStateKey) o;
-      UUIDStateKey that = (UUIDStateKey) o;
+      return Objects.equals(uuid, that.uuid) && stateType == that.stateType;
-      return Objects.equals(uuid, that.uuid);
    }
 
 
@@ -96,7 +107,7 @@ public class UUIDStateKey extends AbstractStateKey
    @Override
    public int hashCode()
    {
-      return Objects.hash(uuid);
+      return Objects.hash(uuid, stateType);
    }
 
 
@@ -107,6 +118,6 @@ public class UUIDStateKey extends AbstractStateKey
    @Override
    public String toString()
    {
-      return uuid.toString();
+      return "{uuid=" + uuid + ", stateType=" + stateType + '}';
    }
 }

src/main/java/com/kingsrook/qqq/backend/core/utils/ExceptionUtils.java

@@ -22,6 +22,10 @@
 package com.kingsrook.qqq.backend.core.utils;
 
 
+import java.util.HashSet;
+import java.util.Set;
+
+
 /*******************************************************************************
  ** Utility class for working with exceptions.
  **
@@ -31,12 +35,12 @@ public class ExceptionUtils
 
    /*******************************************************************************
     ** Find a specific exception class in an exception's caused-by chain.  Returns
-    ** null if not found.  Be aware, checks for class.equals -- not instanceof.
+    ** null if not found.  Be aware, uses class.isInstaance (so sub-classes get found).
     **
     *******************************************************************************/
    public static <T extends Throwable> T findClassInRootChain(Throwable e, Class<T> targetClass)
    {
-      if (e == null)
+      if(e == null)
       {
          return (null);
       }
@@ -54,4 +58,34 @@ public class ExceptionUtils
       return findClassInRootChain(e.getCause(), targetClass);
    }
 
+
+
+   /*******************************************************************************
+    ** Get the root exception in a caused-by-chain.
+    **
+    *******************************************************************************/
+   public static Throwable getRootException(Exception exception)
+   {
+      if(exception == null)
+      {
+         return (null);
+      }
+
+      Throwable      root = exception;
+      Set<Throwable> seen = new HashSet<>();
+      while(root.getCause() != null)
+      {
+         if(seen.contains(root))
+         {
+            //////////////////////////
+            // avoid infinite loops //
+            //////////////////////////
+            break;
+         }
+         seen.add(root);
+         root = root.getCause();
+      }
+
+      return (root);
+   }
 }

src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java

@@ -0,0 +1,125 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.utils;
+
+
+import java.math.BigDecimal;
+import com.kingsrook.qqq.backend.core.exceptions.QValueException;
+
+
+/*******************************************************************************
+ ** Utilities work values - e.g., type-cast-like operations
+ *******************************************************************************/
+public class ValueUtils
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Integer getValueAsInteger(Object value) throws QValueException
+   {
+      try
+      {
+         if(value == null)
+         {
+            return (null);
+         }
+         else if(value instanceof Integer i)
+         {
+            return (i);
+         }
+         else if(value instanceof Long l)
+         {
+            return Math.toIntExact(l);
+         }
+         else if(value instanceof Float f)
+         {
+            if (f.intValue() != f)
+            {
+               throw (new QValueException(f + " does not have an exact integer representation."));
+            }
+            return (f.intValue());
+         }
+         else if(value instanceof Double d)
+         {
+            if (d.intValue() != d)
+            {
+               throw (new QValueException(d + " does not have an exact integer representation."));
+            }
+            return (d.intValue());
+         }
+         else if(value instanceof BigDecimal bd)
+         {
+            return bd.intValueExact();
+         }
+         else if(value instanceof String s)
+         {
+            if(!StringUtils.hasContent(s))
+            {
+               return (null);
+            }
+
+            try
+            {
+               return (Integer.parseInt(s));
+            }
+            catch(NumberFormatException nfe)
+            {
+               if(s.contains(","))
+               {
+                  String sWithoutCommas = s.replaceAll(",", "");
+                  try
+                  {
+                     return (getValueAsInteger(sWithoutCommas));
+                  }
+                  catch(Exception ignore)
+                  {
+                     throw (nfe);
+                  }
+               }
+               if(s.matches(".*\\.\\d+$"))
+               {
+                  String sWithoutDecimal = s.replaceAll("\\.\\d+$", "");
+                  try
+                  {
+                     return (getValueAsInteger(sWithoutDecimal));
+                  }
+                  catch(Exception ignore)
+                  {
+                     throw (nfe);
+                  }
+               }
+               throw (nfe);
+            }
+         }
+         else
+         {
+            throw (new IllegalArgumentException("Unsupported class " + value.getClass().getName() + " for converting to Integer."));
+         }
+      }
+      catch(Exception e)
+      {
+         throw (new QValueException("Value [" + value + "] could not be converted to an Integer.", e));
+      }
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/core/actions/RunBackendStepActionTest.java

@@ -37,7 +37,6 @@ import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
@@ -61,7 +60,6 @@ public class RunBackendStepActionTest
       request.setCallback(callback);
       RunBackendStepResult result = new RunBackendStepAction().execute(request);
       assertNotNull(result);
-      assertNull(result.getError());
       assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("mockValue")), "records should have a mock value");
       assertTrue(result.getValues().containsKey("mockValue"), "result object should have a mock value");
       assertEquals("ABC", result.getValues().get("greetingPrefix"), "result object should have value from our callback");

src/test/java/com/kingsrook/qqq/backend/core/actions/RunProcessTest.java

@@ -26,16 +26,20 @@ import java.io.Serializable;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.callbacks.QProcessCallback;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessRequest;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessResult;
 import com.kingsrook.qqq.backend.core.model.actions.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
@@ -51,14 +55,13 @@ public class RunProcessTest
    @Test
    public void test() throws QException
    {
-      TestCallback callback = new TestCallback();
+      TestCallback      callback = new TestCallback();
-      RunProcessRequest request = new RunProcessRequest(TestUtils.defineInstance());
+      RunProcessRequest request  = new RunProcessRequest(TestUtils.defineInstance());
       request.setSession(TestUtils.getMockSession());
       request.setProcessName("addToPeoplesAge");
       request.setCallback(callback);
       RunProcessResult result = new RunProcessAction().execute(request);
       assertNotNull(result);
-      assertNull(result.getError());
       assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("age")), "records should have a value set by the process");
       assertTrue(result.getValues().containsKey("maxAge"), "process result object should have a value set by the first function in the process");
       assertTrue(result.getValues().containsKey("totalYearsAdded"), "process result object should have a value set by the second function in the process");
@@ -68,6 +71,49 @@ public class RunProcessTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testBackendOnly() throws QException
+   {
+      TestCallback      callback    = new TestCallback();
+      QInstance         instance    = TestUtils.defineInstance();
+      RunProcessRequest request     = new RunProcessRequest(instance);
+      String            processName = TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE;
+
+      request.setSession(TestUtils.getMockSession());
+      request.setProcessName(processName);
+      request.setBackendOnly(true);
+      request.setCallback(callback);
+      RunProcessResult result0 = new RunProcessAction().execute(request);
+
+      assertNotNull(result0);
+
+      ///////////////////////////////////////////////////////////////
+      // make sure we were told that we broke at a (frontend) step //
+      ///////////////////////////////////////////////////////////////
+      Optional<String> breakingAtStep0 = result0.getProcessState().getNextStepName();
+      assertTrue(breakingAtStep0.isPresent());
+      assertInstanceOf(QFrontendStepMetaData.class, instance.getProcessStep(processName, breakingAtStep0.get()));
+
+      //////////////////////////////////////////////
+      // now run again, proceeding from this step //
+      //////////////////////////////////////////////
+      request.setStartAfterStep(breakingAtStep0.get());
+      RunProcessResult result1 = new RunProcessAction().execute(request);
+
+      ////////////////////////////////////////////////////////////////////
+      // make sure we were told that we broke at the next frontend step //
+      ////////////////////////////////////////////////////////////////////
+      Optional<String> breakingAtStep1 = result1.getProcessState().getNextStepName();
+      assertTrue(breakingAtStep1.isPresent());
+      assertInstanceOf(QFrontendStepMetaData.class, instance.getProcessStep(processName, breakingAtStep1.get()));
+      assertNotEquals(breakingAtStep0.get(), breakingAtStep1.get());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -76,6 +122,8 @@ public class RunProcessTest
       private boolean wasCalledForQueryFilter = false;
       private boolean wasCalledForFieldValues = false;
 
+
+
       /*******************************************************************************
        **
        *******************************************************************************/
@@ -96,7 +144,7 @@ public class RunProcessTest
       {
          wasCalledForFieldValues = true;
          Map<String, Serializable> rs = new HashMap<>();
-         if (fields.stream().anyMatch(f -> f.getName().equals("yearsToAdd")))
+         if(fields.stream().anyMatch(f -> f.getName().equals("yearsToAdd")))
          {
             rs.put("yearsToAdd", 42);
          }

src/test/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManagerTest.java

@@ -0,0 +1,162 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.async;
+
+
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+
+/*******************************************************************************
+ ** Unit test for AsyncJobManager
+ *******************************************************************************/
+class AsyncJobManagerTest
+{
+   public static final int ANSWER = 42;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testCompletesInTime() throws JobGoingAsyncException, QException
+   {
+      AsyncJobManager asyncJobManager = new AsyncJobManager();
+      Integer answer = asyncJobManager.startJob(5, TimeUnit.SECONDS, (callback) ->
+      {
+         return (ANSWER);
+      });
+      assertEquals(ANSWER, answer);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testJobGoesAsync()
+   {
+      assertThrows(JobGoingAsyncException.class, () ->
+      {
+         AsyncJobManager asyncJobManager = new AsyncJobManager();
+         Integer answer = asyncJobManager.startJob(1, TimeUnit.MICROSECONDS, (callback) ->
+         {
+            Thread.sleep(1_000);
+            return (ANSWER);
+         });
+      });
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testJobThatThrowsBeforeTimeout()
+   {
+      assertThrows(QException.class, () ->
+      {
+         AsyncJobManager asyncJobManager = new AsyncJobManager();
+         asyncJobManager.startJob(1, TimeUnit.SECONDS, (callback) ->
+         {
+            throw (new IllegalArgumentException("I must throw."));
+         });
+      });
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testJobThatThrowsAfterTimeout() throws QException, InterruptedException
+   {
+      String          message         = "I must throw.";
+      AsyncJobManager asyncJobManager = new AsyncJobManager();
+      try
+      {
+         asyncJobManager.startJob(1, TimeUnit.MILLISECONDS, (callback) ->
+         {
+            Thread.sleep(50);
+            throw (new IllegalArgumentException(message));
+         });
+      }
+      catch(JobGoingAsyncException jgae)
+      {
+         Thread.sleep(100);
+         Optional<AsyncJobStatus> jobStatus = asyncJobManager.getJobStatus(jgae.getJobUUID());
+         assertEquals(AsyncJobState.ERROR, jobStatus.get().getState());
+         assertNotNull(jobStatus.get().getCaughtException());
+         assertEquals(message, jobStatus.get().getCaughtException().getMessage());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testGettingStatusOfAsyncJob() throws InterruptedException, QException
+   {
+      AsyncJobManager asyncJobManager = new AsyncJobManager();
+      String          preMessage      = "Going to sleep";
+      String          postMessage     = "Waking up";
+      try
+      {
+         asyncJobManager.startJob(50, TimeUnit.MILLISECONDS, (callback) ->
+         {
+            callback.updateStatus(preMessage);
+            callback.updateStatus(0, 1);
+            Thread.sleep(100);
+            callback.updateStatus(postMessage, 1, 1);
+            return (ANSWER);
+         });
+      }
+      catch(JobGoingAsyncException jgae)
+      {
+         assertNotNull(jgae.getJobUUID());
+         Optional<AsyncJobStatus> jobStatus = asyncJobManager.getJobStatus(jgae.getJobUUID());
+
+         assertEquals(AsyncJobState.RUNNING, jobStatus.get().getState());
+         assertEquals(preMessage, jobStatus.get().getMessage());
+         assertEquals(0, jobStatus.get().getCurrent());
+         assertEquals(1, jobStatus.get().getTotal());
+
+         Thread.sleep(200);
+         assertEquals(AsyncJobState.COMPLETE, jobStatus.get().getState());
+         assertEquals(postMessage, jobStatus.get().getMessage());
+         assertEquals(1, jobStatus.get().getCurrent());
+         assertEquals(1, jobStatus.get().getTotal());
+      }
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java

@@ -31,7 +31,6 @@ import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
@@ -56,7 +55,6 @@ class BasicETLProcessTest
 
       RunProcessResult result = new RunProcessAction().execute(request);
       assertNotNull(result);
-      assertNull(result.getError());
       assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process");
    }
 
@@ -83,7 +81,6 @@ class BasicETLProcessTest
 
       RunProcessResult result = new RunProcessAction().execute(request);
       assertNotNull(result);
-      assertNull(result.getError());
       assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process");
    }
 

src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java

@@ -42,7 +42,7 @@ public class InMemoryStateProviderTest
    public void testStateNotFound()
    {
       InMemoryStateProvider stateProvider = InMemoryStateProvider.getInstance();
-      UUIDStateKey          key           = new UUIDStateKey();
+      UUIDAndTypeStateKey   key           = new UUIDAndTypeStateKey(StateType.PROCESS_STATUS);
 
       Assertions.assertTrue(stateProvider.get(QRecord.class, key).isEmpty(), "Key not found in state should return empty");
    }
@@ -55,7 +55,7 @@ public class InMemoryStateProviderTest
    public void testSimpleStateFound()
    {
       InMemoryStateProvider stateProvider = InMemoryStateProvider.getInstance();
-      UUIDStateKey          key           = new UUIDStateKey();
+      UUIDAndTypeStateKey   key           = new UUIDAndTypeStateKey(StateType.PROCESS_STATUS);
 
       String uuid = UUID.randomUUID().toString();
       QRecord qRecord = new QRecord().withValue("uuid", uuid);
@@ -74,7 +74,7 @@ public class InMemoryStateProviderTest
    public void testWrongTypeOnGet()
    {
       InMemoryStateProvider stateProvider = InMemoryStateProvider.getInstance();
-      UUIDStateKey          key           = new UUIDStateKey();
+      UUIDAndTypeStateKey   key           = new UUIDAndTypeStateKey(StateType.PROCESS_STATUS);
 
       String uuid = UUID.randomUUID().toString();
       QRecord qRecord = new QRecord().withValue("uuid", uuid);

src/test/java/com/kingsrook/qqq/backend/core/state/TempFileStateProviderTest.java

@@ -42,7 +42,7 @@ public class TempFileStateProviderTest
    public void testStateNotFound()
    {
       TempFileStateProvider stateProvider = TempFileStateProvider.getInstance();
-      UUIDStateKey          key           = new UUIDStateKey();
+      UUIDAndTypeStateKey   key           = new UUIDAndTypeStateKey(StateType.PROCESS_STATUS);
 
       Assertions.assertTrue(stateProvider.get(QRecord.class, key).isEmpty(), "Key not found in state should return empty");
    }
@@ -55,7 +55,7 @@ public class TempFileStateProviderTest
    public void testSimpleStateFound()
    {
       TempFileStateProvider stateProvider = TempFileStateProvider.getInstance();
-      UUIDStateKey          key           = new UUIDStateKey();
+      UUIDAndTypeStateKey   key           = new UUIDAndTypeStateKey(StateType.PROCESS_STATUS);
 
       String uuid = UUID.randomUUID().toString();
       QRecord qRecord = new QRecord().withValue("uuid", uuid);
@@ -74,7 +74,7 @@ public class TempFileStateProviderTest
    public void testWrongTypeOnGet()
    {
       TempFileStateProvider stateProvider = TempFileStateProvider.getInstance();
-      UUIDStateKey          key           = new UUIDStateKey();
+      UUIDAndTypeStateKey   key           = new UUIDAndTypeStateKey(StateType.PROCESS_STATUS);
 
       String uuid = UUID.randomUUID().toString();
       QRecord qRecord = new QRecord().withValue("uuid", uuid);

src/test/java/com/kingsrook/qqq/backend/core/utils/ExceptionUtilsTest.java

@@ -25,7 +25,8 @@ package com.kingsrook.qqq.backend.core.utils;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import org.junit.jupiter.api.Test;
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
 
 
 /*******************************************************************************
@@ -39,13 +40,87 @@ class ExceptionUtilsTest
     **
     *******************************************************************************/
    @Test
-   void findClassInRootChain()
+   void testFindClassInRootChain()
    {
       assertNull(ExceptionUtils.findClassInRootChain(null, QUserFacingException.class));
-      QUserFacingException target = new QUserFacingException("target");
 
+      QUserFacingException target = new QUserFacingException("target");
       assertSame(target, ExceptionUtils.findClassInRootChain(target, QUserFacingException.class));
       assertSame(target, ExceptionUtils.findClassInRootChain(new QException("decoy", target), QUserFacingException.class));
       assertNull(ExceptionUtils.findClassInRootChain(new QException("decoy", target), IllegalArgumentException.class));
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetRootException()
+   {
+      assertNull(ExceptionUtils.getRootException(null));
+
+      Exception root = new Exception("root");
+      assertSame(root, ExceptionUtils.getRootException(root));
+
+      Exception container = new Exception("container", root);
+      assertSame(root, ExceptionUtils.getRootException(container));
+
+      Exception middle = new Exception("middle", root);
+      Exception top = new Exception("top", middle);
+      assertSame(root, ExceptionUtils.getRootException(top));
+
+      ////////////////////////////////////////////////////////////////////////////////////////
+      // without the code that checks for loops, these next two checks cause infinite loops //
+      ////////////////////////////////////////////////////////////////////////////////////////
+      MyException selfCaused = new MyException("selfCaused");
+      selfCaused.setCause(selfCaused);
+      assertSame(selfCaused, ExceptionUtils.getRootException(selfCaused));
+
+      MyException cycle1 = new MyException("cycle1");
+      MyException cycle2 = new MyException("cycle2");
+      cycle1.setCause(cycle2);
+      cycle2.setCause(cycle1);
+      assertSame(cycle1, ExceptionUtils.getRootException(cycle1));
+      assertSame(cycle2, ExceptionUtils.getRootException(cycle2));
+   }
+
+
+
+   /*******************************************************************************
+    ** Test exception class - lets you set the cause, easier to create a loop.
+    *******************************************************************************/
+   public class MyException extends Exception
+   {
+      private Throwable myCause = null;
+
+
+
+      public MyException(String message)
+      {
+         super(message);
+      }
+
+
+
+      public MyException(Throwable cause)
+      {
+         super(cause);
+      }
+
+
+
+      public void setCause(Throwable cause)
+      {
+         myCause = cause;
+      }
+
+
+
+      @Override
+      public synchronized Throwable getCause()
+      {
+         return (myCause);
+      }
+   }
 }

src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.utils;
 
 
 import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge;
+import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.GetAgeStatistics;
 import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
 import com.kingsrook.qqq.backend.core.interfaces.mock.MockBackendStep;
 import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData;
@@ -44,6 +46,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.modules.mock.MockAuthenticationModule;
+import com.kingsrook.qqq.backend.core.modules.mock.MockBackendModule;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
 
 
@@ -118,7 +121,7 @@ public class TestUtils
    {
       return new QBackendMetaData()
          .withName(DEFAULT_BACKEND_NAME)
-         .withBackendType("mock");
+         .withBackendType(MockBackendModule.class);
    }
 
 
@@ -189,7 +192,7 @@ public class TestUtils
             .withCode(new QCodeReference()
                .withName(MockBackendStep.class.getName())
                .withCodeType(QCodeType.JAVA)
-               .withCodeUsage(QCodeUsage.FUNCTION)) // todo - needed, or implied in this context?
+               .withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context?
             .withInputData(new QFunctionInputMetaData()
                .withRecordListMetaData(new QRecordListMetaData().withTableName("person"))
                .withFieldList(List.of(
@@ -227,7 +230,7 @@ public class TestUtils
             .withCode(new QCodeReference()
                .withName(MockBackendStep.class.getName())
                .withCodeType(QCodeType.JAVA)
-               .withCodeUsage(QCodeUsage.FUNCTION)) // todo - needed, or implied in this context?
+               .withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context?
             .withInputData(new QFunctionInputMetaData()
                .withRecordListMetaData(new QRecordListMetaData().withTableName("person"))
                .withFieldList(List.of(
@@ -266,9 +269,9 @@ public class TestUtils
          .addStep(new QBackendStepMetaData()
             .withName("getAgeStatistics")
             .withCode(new QCodeReference()
-               .withName("com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.GetAgeStatistics")
+               .withName(GetAgeStatistics.class.getName())
                .withCodeType(QCodeType.JAVA)
-               .withCodeUsage(QCodeUsage.FUNCTION))
+               .withCodeUsage(QCodeUsage.BACKEND_STEP))
             .withInputData(new QFunctionInputMetaData()
                .withRecordListMetaData(new QRecordListMetaData().withTableName("person")))
             .withOutputMetaData(new QFunctionOutputMetaData()
@@ -281,9 +284,9 @@ public class TestUtils
          .addStep(new QBackendStepMetaData()
             .withName("addAge")
             .withCode(new QCodeReference()
-               .withName("com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge")
+               .withName(AddAge.class.getName())
                .withCodeType(QCodeType.JAVA)
-               .withCodeUsage(QCodeUsage.FUNCTION))
+               .withCodeUsage(QCodeUsage.BACKEND_STEP))
             .withInputData(new QFunctionInputMetaData()
                .withFieldList(List.of(new QFieldMetaData("yearsToAdd", QFieldType.INTEGER))))
             .withOutputMetaData(new QFunctionOutputMetaData()

src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java

@@ -0,0 +1,66 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.utils;
+
+
+import java.math.BigDecimal;
+import com.kingsrook.qqq.backend.core.exceptions.QValueException;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.*;
+
+
+/*******************************************************************************
+ ** Unit test for ValueUtils
+ *******************************************************************************/
+class ValueUtilsTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetValueAsInteger() throws QValueException
+   {
+      assertNull(ValueUtils.getValueAsInteger(null));
+      assertNull(ValueUtils.getValueAsInteger(""));
+      assertNull(ValueUtils.getValueAsInteger(" "));
+      assertEquals(1, ValueUtils.getValueAsInteger(1));
+      assertEquals(1, ValueUtils.getValueAsInteger("1"));
+      assertEquals(1_000, ValueUtils.getValueAsInteger("1,000"));
+      assertEquals(1_000_000, ValueUtils.getValueAsInteger("1,000,000"));
+      assertEquals(1, ValueUtils.getValueAsInteger(new BigDecimal(1)));
+      assertEquals(1, ValueUtils.getValueAsInteger(new BigDecimal("1.00")));
+      assertEquals(-1, ValueUtils.getValueAsInteger("-1.00"));
+      assertEquals(1_000, ValueUtils.getValueAsInteger("1,000.00"));
+      assertEquals(1_000, ValueUtils.getValueAsInteger(1000L));
+      assertEquals(1, ValueUtils.getValueAsInteger(1.0F));
+      assertEquals(1, ValueUtils.getValueAsInteger(1.0D));
+
+      assertThrows(QValueException.class, () -> ValueUtils.getValueAsInteger("a"));
+      assertThrows(QValueException.class, () -> ValueUtils.getValueAsInteger("a,b"));
+      assertThrows(QValueException.class, () -> ValueUtils.getValueAsInteger(new Object()));
+      assertThrows(QValueException.class, () -> ValueUtils.getValueAsInteger(1_000_000_000_000L));
+      assertThrows(QValueException.class, () -> ValueUtils.getValueAsInteger(1.1F));
+      assertThrows(QValueException.class, () -> ValueUtils.getValueAsInteger(1.1D));
+   }
+
+}