QQQ-14 feedback from code review

src/main/java/com/kingsrook/qqq/backend/core/actions/RunFunctionAction.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.callbacks.QProcessCallback;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
@@ -54,9 +55,6 @@ public class RunFunctionAction
    {
       ActionHelper.validateSession(runFunctionRequest);
 
-      ///////////////////////////////////////////////////////
-      //
-      ///////////////////////////////////////////////////////
       QProcessMetaData process = runFunctionRequest.getInstance().getProcess(runFunctionRequest.getProcessName());
       if(process == null)
       {
@@ -88,10 +86,10 @@ public class RunFunctionAction
     ** via the callback
     **
     *******************************************************************************/
-   private void ensureInputFieldsAreInRequest(RunFunctionRequest runFunctionRequest, QFunctionMetaData function)
+   private void ensureInputFieldsAreInRequest(RunFunctionRequest runFunctionRequest, QFunctionMetaData function) throws QException
    {
       QFunctionInputMetaData inputMetaData = function.getInputMetaData();
-      if (inputMetaData == null)
+      if(inputMetaData == null)
       {
          return;
       }
@@ -109,7 +107,13 @@ public class RunFunctionAction
 
       if(!fieldsToGet.isEmpty())
       {
-         Map<String, Serializable> fieldValues = runFunctionRequest.getCallback().getFieldValues(fieldsToGet);
+         QProcessCallback callback = runFunctionRequest.getCallback();
+         if(callback == null)
+         {
+            throw (new QException("Function is missing values for fields, but no callback was present to request fields from a user"));
+         }
+
+         Map<String, Serializable> fieldValues = callback.getFieldValues(fieldsToGet);
          for(Map.Entry<String, Serializable> entry : fieldValues.entrySet())
          {
             runFunctionRequest.addValue(entry.getKey(), entry.getValue());
@@ -138,7 +142,14 @@ public class RunFunctionAction
             // todo - handle this being async (e.g., http)
             // seems like it just needs to throw, breaking this flow, and to send a response to the frontend, directing it to prompt the user for the needed data
             // then this function can re-run, hopefully with the needed data.
-            queryRequest.setFilter(runFunctionRequest.getCallback().getQueryFilter());
+
+            QProcessCallback callback = runFunctionRequest.getCallback();
+            if(callback == null)
+            {
+               throw (new QException("Function is missing input records, but no callback was present to get a query filter from a user"));
+            }
+
+            queryRequest.setFilter(callback.getQueryFilter());
 
             QueryResult queryResult = new QueryAction().execute(queryRequest);
             runFunctionRequest.setRecords(queryResult.getRecords());

src/main/java/com/kingsrook/qqq/backend/core/actions/RunProcessAction.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions;
 
 
 import java.util.List;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
@@ -145,10 +146,11 @@ public class RunProcessAction
     ** Load the process state into a function request from the state provider
     **
     *******************************************************************************/
-   private void loadState(UUIDStateKey stateKey, RunFunctionRequest runFunctionRequest)
+   private void loadState(UUIDStateKey stateKey, RunFunctionRequest runFunctionRequest) throws QException
    {
-      ProcessState processState = getStateProvider().get(ProcessState.class, stateKey);
+      Optional<ProcessState> processState = getStateProvider().get(ProcessState.class, stateKey);
-      runFunctionRequest.seedFromProcessState(processState);
+      runFunctionRequest.seedFromProcessState(processState
+         .orElseThrow(() -> new QException("Could not find process state in state provider.")));
    }
 
 }

src/main/java/com/kingsrook/qqq/backend/core/callbacks/NoopCallback.java

@@ -1,58 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.callbacks;
-
-
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import com.kingsrook.qqq.backend.core.model.actions.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
-
-
-/*******************************************************************************
- ** Simple implementation of a callback, that does no-op (returns empty objects).
- ** Useful for scaffolding, perhaps tests.
- *******************************************************************************/
-public class NoopCallback implements QProcessCallback
-{
-   /*******************************************************************************
-    ** Get the filter query for this callback.
-    *******************************************************************************/
-   @Override
-   public QQueryFilter getQueryFilter()
-   {
-      return (new QQueryFilter());
-   }
-
-
-
-   /*******************************************************************************
-    ** Get the field values for this callback.
-    *******************************************************************************/
-   @Override
-   public Map<String, Serializable> getFieldValues(List<QFieldMetaData> fields)
-   {
-      return (Collections.emptyMap());
-   }
-}

src/main/java/com/kingsrook/qqq/backend/core/interfaces/FunctionBody.java

@@ -28,12 +28,17 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
 
 
 /*******************************************************************************
- ** TODO - document!
+ ** Simple interface that a "custom" function (as in, a component of a Process)
+ ** must implement.
  *******************************************************************************/
 public interface FunctionBody
 {
    /*******************************************************************************
-    ** TODO - document!
+    ** Execute the function - using the request as input, and the result as output.
+    **
+    ** TODO - think about - why take the Result object as a param, instead of return it?
+    **  Is this way easier for inter-language operability maybe?
+    *   Also - there's way too much "process-specific gunk" in the Request object - can we simplify it?
     *******************************************************************************/
    void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException;
 }

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java

@@ -42,6 +42,7 @@ public class QBackendMetaData
    // @JsonFilter("secretsFilter")
 
 
+
    /*******************************************************************************
     ** Default Constructor.
     *******************************************************************************/
@@ -51,17 +52,6 @@ public class QBackendMetaData
 
 
 
-   /*******************************************************************************
-    ** Copy Constructor.  Meant for use by sub-classes.  Should copy all fields!
-    *******************************************************************************/
-   protected QBackendMetaData(QBackendMetaData source)
-   {
-      this.name = source.name;
-      this.backendType = source.backendType;
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -83,7 +73,7 @@ public class QBackendMetaData
 
 
    /*******************************************************************************
-    **
+    ** Fluent setter, returning generically, to help sub-class fluent flows
     *******************************************************************************/
    @SuppressWarnings("unchecked")
    public <T extends QBackendMetaData> T withName(String name)
@@ -136,7 +126,7 @@ public class QBackendMetaData
 
 
    /*******************************************************************************
-    **
+    ** Fluent setter, returning generically, to help sub-class fluent flows
     *******************************************************************************/
    @SuppressWarnings("unchecked")
    public <T extends QBackendMetaData> T withBackendType(String backendType)

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java

@@ -59,7 +59,7 @@ public class QInstance
 
 
    /*******************************************************************************
-    **
+    ** Get the backend for a given table name
     *******************************************************************************/
    public QBackendMetaData getBackendForTable(String tableName)
    {
@@ -69,19 +69,16 @@ public class QInstance
          throw (new IllegalArgumentException("No table with name [" + tableName + "] found in this instance."));
       }
 
-      QBackendMetaData backend = backends.get(table.getBackendName());
-
       //////////////////////////////////////////////////////////////////////////////////////////////
       // validation should already let us know that this is valid, so no need to check/throw here //
       //////////////////////////////////////////////////////////////////////////////////////////////
-
+      return (backends.get(table.getBackendName()));
-      return (backend);
    }
 
 
 
    /*******************************************************************************
-    **
+    ** Get the list of processes associated with a given table name
     *******************************************************************************/
    public List<QProcessMetaData> getProcessesForTable(String tableName)
    {

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/DeserializerUtils.java

@@ -38,19 +38,30 @@ import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
 import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
- **
+ ** Utility methods to help with deserializing JSON streams into QQQ models.
+ ** Specifically meant to be used within a jackson custom deserializer (e.g.,
+ ** an implementation of JsonDeserializer).
  *******************************************************************************/
 public class DeserializerUtils
 {
+   private static final Logger LOG = LogManager.getLogger(DeserializerUtils.class);
+
+
 
    /*******************************************************************************
-    **
+    ** For a given (jackson, JSON) treeNode, look at its backendType property,
+    ** and return an instance of the corresponding QBackendModule.
     *******************************************************************************/
    public static QBackendModuleInterface getBackendModule(TreeNode treeNode) throws IOException
    {
+      /////////////////////////////////////////////////////////////////////////////////
+      // validate the backendType property is present, as text, in the json treeNode //
+      /////////////////////////////////////////////////////////////////////////////////
       TreeNode backendTypeTreeNode = treeNode.get("backendType");
       if(backendTypeTreeNode == null || backendTypeTreeNode instanceof NullNode)
       {
@@ -61,44 +72,67 @@ public class DeserializerUtils
       {
          throw new IOException("backendType is not a string value (is: " + backendTypeTreeNode.getClass().getSimpleName() + ")");
       }
-      else
-      {
-         String backendType = textNode.asText();
 
-         try
+      try
-         {
+      {
-            return new QBackendModuleDispatcher().getQBackendModule(backendType);
+         /////////////////////////////////////////////////////////////////////////////////////////////////
-         }
+         // get the value of the backendType json node, and use it to look up the qBackendModule object //
-         catch(QModuleDispatchException e)
+         /////////////////////////////////////////////////////////////////////////////////////////////////
-         {
+         String backendType = textNode.asText();
-            throw (new IOException(e));
+         return new QBackendModuleDispatcher().getQBackendModule(backendType);
-         }
+      }
+      catch(QModuleDispatchException e)
+      {
+         throw (new IOException(e));
       }
    }
 
 
 
    /*******************************************************************************
+    ** Using reflection, create & populate an instance of a class, based on the
+    ** properties in a jackson/json treeNode.
     **
     *******************************************************************************/
    public static <T> T reflectivelyDeserialize(Class<T> outputClass, TreeNode treeNode) throws IOException
    {
       try
       {
+         /////////////////////////////////
+         // construct the output object //
+         /////////////////////////////////
          T output = outputClass.getConstructor().newInstance();
 
+         /////////////////////////////////////////////////////////////////////////////////////////////////
+         // set up a mapping between field names, and lambdas which will take a String (from the json), //
+         // and set it in the output object, doing type conversion as needed.                           //
+         // do this by iterating over methods on the output class that look like setters.               //
+         /////////////////////////////////////////////////////////////////////////////////////////////////
          Map<String, Consumer<String>> setterMap = new HashMap<>();
          for(Method method : outputClass.getMethods())
          {
+            /////////////////////////////////////////////////////////////
+            // setters start with the word "set", and have 1 parameter //
+            /////////////////////////////////////////////////////////////
             if(method.getName().startsWith("set") && method.getParameterTypes().length == 1)
             {
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // get the parameter type, and the name of the field (remove set from the method name, and downshift the first letter) //
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
                Class<?> parameterType = method.getParameterTypes()[0];
                String   fieldName     = method.getName().substring(3, 4).toLowerCase(Locale.ROOT) + method.getName().substring(4);
 
+               ///////////////////////////////////////////////////////////////////////////////////
+               // put the entry in the map - where the value here is a consumer lambda function //
+               ///////////////////////////////////////////////////////////////////////////////////
                setterMap.put(fieldName, (String value) ->
                {
                   try
                   {
+                     //////////////////////////////////////////////////////////////////////////////////////////////////
+                     // based on the parameter type, handle it differently - either type-converting (e.g., parseInt) //
+                     // or gracefully ignoring, or failing.                                                          //
+                     //////////////////////////////////////////////////////////////////////////////////////////////////
                      if(parameterType.equals(String.class))
                      {
                         method.invoke(output, value);
@@ -121,23 +155,20 @@ public class DeserializerUtils
                      }
                      else if(parameterType.equals(Class.class))
                      {
-                        ////////////////////////////////////////////////////////////
+                        ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
-                        // specifically do NOT try to handle Class type arguments //
+                        // specifically do NOT try to handle Class type arguments                                                    //
-                        ////////////////////////////////////////////////////////////
+                        // we hit this when trying to de-serialize a QBackendMetaData, and we found its setBackendType(Class) method //
+                        ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
                      }
                      else if(parameterType.getPackageName().startsWith("java."))
                      {
-                        ////////////////////////////////////////////////////////////////////////
+                        /////////////////////////////////////////////////////////////////////////////////////////////////////////
-                        // if we hit this, we might want to add an else-if to handle the type //
+                        // if we hit this, we might want to add an else-if to handle the type.                                 //
-                        ////////////////////////////////////////////////////////////////////////
+                        // otherwise, either find some jackson annotation that makes sense, and apply it to the setter method, //
+                        // or if no jackson annotation is right, then come up with annotation of our own.                      //
+                        /////////////////////////////////////////////////////////////////////////////////////////////////////////
                         throw (new RuntimeException("Field " + fieldName + " is of an unhandled type " + parameterType.getName() + " when deserializing " + outputClass.getName()));
                      }
-                     else
-                     {
-                        ////////////////////////////////////
-                        // gracefully ignore other types. //
-                        ////////////////////////////////////
-                     }
                   }
                   catch(IllegalAccessException | InvocationTargetException e)
                   {
@@ -153,9 +184,8 @@ public class DeserializerUtils
       }
       catch(Exception e)
       {
-         throw (new IOException("Error reflectively deserializing table details", e));
+         throw (new IOException("Error deserializing json object into instance of " + outputClass.getName(), e));
       }
-
    }
 
 
@@ -169,24 +199,35 @@ public class DeserializerUtils
     *******************************************************************************/
    private static void deserializeBean(TreeNode treeNode, Map<String, Consumer<String>> setterMap) throws IOException
    {
+      ///////////////////////////////////////////////////////
+      // iterate over fields in the json object (treeNode) //
+      ///////////////////////////////////////////////////////
       Iterator<String> fieldNamesIterator = treeNode.fieldNames();
       while(fieldNamesIterator.hasNext())
       {
          String fieldName = fieldNamesIterator.next();
 
+         //////////////////////////////////////////////////////////////////////////
+         // error if we find a field in the json that we don't have a setter for //
+         //////////////////////////////////////////////////////////////////////////
          if(!setterMap.containsKey(fieldName))
          {
-            throw (new IllegalArgumentException("Unexpected value: " + fieldName));
+            throw (new IOException("Unexpected field (no corresponding setter): " + fieldName));
          }
 
+         // call the setter -
          TreeNode fieldNode = treeNode.get(fieldName);
          if(fieldNode instanceof NullNode)
          {
             setterMap.get(fieldName).accept(null);
          }
+         else if(fieldNode instanceof TextNode textNode)
+         {
+            setterMap.get(fieldName).accept(textNode.asText());
+         }
          else
          {
-            setterMap.get(fieldName).accept(((TextNode) fieldNode).asText());
+            throw (new IOException("Unexpected node type (" + fieldNode.getClass() + ") for field: " + fieldName));
          }
       }
    }

src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLCallback.java

@@ -1,73 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
-
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import com.kingsrook.qqq.backend.core.callbacks.QProcessCallback;
-import com.kingsrook.qqq.backend.core.model.actions.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
-
-
-/*******************************************************************************
- ** Provide callback functionality for the BasicETL process
- *******************************************************************************/
-public class BasicETLCallback implements QProcessCallback
-{
-
-   /*******************************************************************************
-    ** Get the filter query for this callback.
-    *******************************************************************************/
-   @Override
-   public QQueryFilter getQueryFilter()
-   {
-      // todo - possibly get something from params?  through state?  added as a method arg?
-      return null;
-   }
-
-
-
-   /*******************************************************************************
-    ** Get the field values for this callback.
-    *******************************************************************************/
-   @SuppressWarnings("checkstyle:Indentation")
-   @Override
-   public Map<String, Serializable> getFieldValues(List<QFieldMetaData> fields)
-   {
-      Map<String, Serializable> rs = new HashMap<>();
-      for(QFieldMetaData field : fields)
-      {
-         // TODO - replace this whole thing with our params mechanism
-         // TODO - add default methods to the interface that throw, presumably?
-         rs.put(field.getName(), switch(field.getName())
-            {
-               case BasicETLProcess.FIELD_SOURCE_TABLE -> "personFile";
-               case BasicETLProcess.FIELD_DESTINATION_TABLE -> "person";
-               default -> throw new IllegalArgumentException("Unhandled field: " + field.getName());
-            });
-      }
-      return (rs);
-   }
-}

src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java

@@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
  *******************************************************************************/
 public class BasicETLProcess
 {
+   public static final String PROCESS_NAME            = "etl.basic";
    public static final String FIELD_SOURCE_TABLE      = "sourceTable";
    public static final String FIELD_DESTINATION_TABLE = "destinationTable";
    public static final String FIELD_RECORD_COUNT      = "recordCount";
@@ -70,7 +71,7 @@ public class BasicETLProcess
             .addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER)));
 
       return new QProcessMetaData()
-         .withName("etl.basic")
+         .withName(PROCESS_NAME)
          .addFunction(extractFunction)
          .addFunction(loadFunction);
    }

src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.state;
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 
 
 /*******************************************************************************
@@ -63,7 +64,7 @@ public class InMemoryStateProvider implements StateProviderInterface
 
 
    /*******************************************************************************
-    **
+    ** Put a block of data, under a key, into the state store.
     *******************************************************************************/
    @Override
    public <T extends Serializable> void put(AbstractStateKey key, T data)
@@ -74,14 +75,14 @@ public class InMemoryStateProvider implements StateProviderInterface
 
 
    /*******************************************************************************
-    **
+    ** Get a block of data, under a key, from the state store.
     *******************************************************************************/
    @Override
-   public <T extends Serializable> T get(Class<? extends T> type, AbstractStateKey key)
+   public <T extends Serializable> Optional<T> get(Class<? extends T> type, AbstractStateKey key)
    {
       try
       {
-         return type.cast(map.get(key));
+         return Optional.ofNullable(type.cast(map.get(key)));
       }
       catch(ClassCastException cce)
       {

src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java

@@ -23,10 +23,22 @@ package com.kingsrook.qqq.backend.core.state;
 
 
 import java.io.Serializable;
+import java.util.Optional;
 
 
 /*******************************************************************************
+ ** QQQ state provider interface.  Provides standard interface for various
+ ** implementations of how to store & retrieve user/process state data, like
+ ** sessions, or process data.  Not like permanent record data - that is done in
+ ** Backend modules.
  **
+ ** Different implementations may be:  in-memory (non-persistent!!), or on-disk
+ ** (with the tradeoffs that has), in-database, in-cache-system, etc.
+ **
+ ** Things which probably haven't been thought about here include:
+ ** - multi-layering.  e.g., always have an in-memory layer on top of a more
+ **   persistent backend, but then how to avoid staleness in-memory?
+ *  - cleanup.  when do we ever purge things to avoid running out of memory/storage?
  *******************************************************************************/
 public interface StateProviderInterface
 {
@@ -39,5 +51,5 @@ public interface StateProviderInterface
    /*******************************************************************************
     ** Get a block of data, under a key, from the state store.
     *******************************************************************************/
-   <T extends Serializable> T get(Class<? extends T> type, AbstractStateKey key);
+   <T extends Serializable> Optional<T> get(Class<? extends T> type, AbstractStateKey key);
 }

src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java

@@ -26,6 +26,7 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import org.apache.commons.io.FileUtils;
 
@@ -63,7 +64,7 @@ public class TempFileStateProvider implements StateProviderInterface
 
 
    /*******************************************************************************
-    **
+    ** Put a block of data, under a key, into the state store.
     *******************************************************************************/
    @Override
    public <T extends Serializable> void put(AbstractStateKey key, T data)
@@ -83,19 +84,19 @@ public class TempFileStateProvider implements StateProviderInterface
 
 
    /*******************************************************************************
-    **
+    ** Get a block of data, under a key, from the state store.
     *******************************************************************************/
    @Override
-   public <T extends Serializable> T get(Class<? extends T> type, AbstractStateKey key)
+   public <T extends Serializable> Optional<T> get(Class<? extends T> type, AbstractStateKey key)
    {
       try
       {
          String json = FileUtils.readFileToString(new File("/tmp/" + key.toString()));
-         return JsonUtils.toObject(json, type);
+         return (Optional.of(JsonUtils.toObject(json, type)));
       }
       catch(FileNotFoundException fnfe)
       {
-         return (null);
+         return (Optional.empty());
       }
       catch(IOException ie)
       {

src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java

@@ -26,11 +26,6 @@ import com.kingsrook.qqq.backend.core.actions.RunProcessAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessRequest;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessResult;
-import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -50,17 +45,12 @@ class BasicETLProcessTest
    @Test
    public void test() throws QException
    {
-      BasicETLProcess   basicETLProcess = new BasicETLProcess();
+      RunProcessRequest request = new RunProcessRequest(TestUtils.defineInstance());
-      QProcessMetaData  processMetaData = basicETLProcess.defineProcessMetaData();
-      QInstance         instance        = TestUtils.defineInstance();
-      RunProcessRequest request         = new RunProcessRequest(instance);
-
-      instance.addProcess(processMetaData);
-      defineFileBackendAndPersonFileTable(instance);
-
       request.setSession(TestUtils.getMockSession());
-      request.setProcessName(processMetaData.getName());
+      request.setProcessName(BasicETLProcess.PROCESS_NAME);
-      request.setCallback(new BasicETLCallback()); // todo - uh, maybe a method on the process to get its callback?
+      request.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.defineTablePerson().getName());
+      request.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.definePersonFileTable().getName());
+
       RunProcessResult result = new RunProcessAction().execute(request);
       assertNotNull(result);
       assertNull(result.getError());
@@ -68,27 +58,4 @@ class BasicETLProcessTest
    }
 
 
-
-   /*******************************************************************************
-    ** Define the 'person' table used in standard tests.
-    *******************************************************************************/
-   public static void defineFileBackendAndPersonFileTable(QInstance instance)
-   {
-      QTableMetaData personFileTable = new QTableMetaData()
-         .withName("personFile")
-         .withLabel("Person File")
-         .withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
-         .withPrimaryKeyField("id")
-         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
-         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))
-         .withField(new QFieldMetaData("firstName", QFieldType.STRING))
-         .withField(new QFieldMetaData("lastName", QFieldType.STRING))
-         .withField(new QFieldMetaData("birthDate", QFieldType.DATE))
-         .withField(new QFieldMetaData("email", QFieldType.STRING))
-         .withField(new QFieldMetaData("homeState", QFieldType.STRING).withPossibleValueSourceName("state"));
-
-      instance.addTable(personFileTable);
-   }
-
 }

src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java

@@ -44,7 +44,7 @@ public class InMemoryStateProviderTest
       InMemoryStateProvider stateProvider = InMemoryStateProvider.getInstance();
       UUIDStateKey          key           = new UUIDStateKey();
 
-      Assertions.assertNull(stateProvider.get(QRecord.class, key), "Key not found in state should return null");
+      Assertions.assertTrue(stateProvider.get(QRecord.class, key).isEmpty(), "Key not found in state should return empty");
    }
 
 
@@ -61,7 +61,7 @@ public class InMemoryStateProviderTest
       QRecord qRecord = new QRecord().withValue("uuid", uuid);
       stateProvider.put(key, qRecord);
 
-      QRecord qRecordFromState = stateProvider.get(QRecord.class, key);
+      QRecord qRecordFromState = stateProvider.get(QRecord.class, key).get();
       Assertions.assertEquals(uuid, qRecordFromState.getValueString("uuid"), "Should read value from state persistence");
    }
 

src/test/java/com/kingsrook/qqq/backend/core/state/TempFileStateProviderTest.java

@@ -44,7 +44,7 @@ public class TempFileStateProviderTest
       TempFileStateProvider stateProvider = TempFileStateProvider.getInstance();
       UUIDStateKey          key           = new UUIDStateKey();
 
-      Assertions.assertNull(stateProvider.get(QRecord.class, key), "Key not found in state should return null");
+      Assertions.assertTrue(stateProvider.get(QRecord.class, key).isEmpty(), "Key not found in state should return empty");
    }
 
 
@@ -61,7 +61,7 @@ public class TempFileStateProviderTest
       QRecord qRecord = new QRecord().withValue("uuid", uuid);
       stateProvider.put(key, qRecord);
 
-      QRecord qRecordFromState = stateProvider.get(QRecord.class, key);
+      QRecord qRecordFromState = stateProvider.get(QRecord.class, key).get();
       Assertions.assertEquals(uuid, qRecordFromState.getValueString("uuid"), "Should read value from state persistence");
    }
 

src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.utils;
 
 
 import java.util.List;
+import com.kingsrook.qqq.backend.core.interfaces.mock.MockFunctionBody;
 import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
@@ -43,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaDa
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListView;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.modules.mock.MockAuthenticationModule;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
 
 
 /*******************************************************************************
@@ -53,6 +55,8 @@ public class TestUtils
 {
    public static String DEFAULT_BACKEND_NAME = "default";
 
+
+
    /*******************************************************************************
     ** Define the instance used in standard tests.
     **
@@ -63,9 +67,11 @@ public class TestUtils
       qInstance.setAuthentication(defineAuthentication());
       qInstance.addBackend(defineBackend());
       qInstance.addTable(defineTablePerson());
+      qInstance.addTable(definePersonFileTable());
       qInstance.addPossibleValueSource(defineStatesPossibleValueSource());
       qInstance.addProcess(defineProcessGreetPeople());
       qInstance.addProcess(defineProcessAddToPeoplesAge());
+      qInstance.addProcess(new BasicETLProcess().defineProcessMetaData());
       return (qInstance);
    }
 
@@ -118,7 +124,7 @@ public class TestUtils
       return new QTableMetaData()
          .withName("person")
          .withLabel("Person")
-         .withBackendName(defineBackend().getName())
+         .withBackendName(DEFAULT_BACKEND_NAME)
          .withPrimaryKeyField("id")
          .withField(new QFieldMetaData("id", QFieldType.INTEGER))
          .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
@@ -132,6 +138,21 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    ** Define a 2nd version of the 'person' table for this test (pretend it's backed by a file)
+    *******************************************************************************/
+   public static QTableMetaData definePersonFileTable()
+   {
+      return (new QTableMetaData()
+         .withName("personFile")
+         .withLabel("Person File")
+         .withBackendName(DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
+         .withFields(TestUtils.defineTablePerson().getFields()));
+   }
+
+
+
    /*******************************************************************************
     ** Define the 'greet people' process
     *******************************************************************************/
@@ -143,7 +164,7 @@ public class TestUtils
          .addFunction(new QFunctionMetaData()
             .withName("prepare")
             .withCode(new QCodeReference()
-               .withName("com.kingsrook.qqq.backend.core.interfaces.mock.MockFunctionBody")
+               .withName(MockFunctionBody.class.getName())
                .withCodeType(QCodeType.JAVA)
                .withCodeUsage(QCodeUsage.FUNCTION)) // todo - needed, or implied in this context?
             .withInputData(new QFunctionInputMetaData()