QQQ-14 add customizer functions; add log4j config; add SecretReader;

pom.xml

@@ -71,6 +71,12 @@
          <artifactId>commons-csv</artifactId>
          <version>1.8</version>
       </dependency>
+      <dependency>
+         <groupId>org.assertj</groupId>
+         <artifactId>assertj-core</artifactId>
+         <version>3.23.1</version>
+         <scope>test</scope>
+      </dependency>
 
       <!-- Common deps for all qqq modules -->
       <dependency>

src/main/java/com/kingsrook/qqq/backend/core/actions/InsertAction.java

@@ -27,6 +27,8 @@ import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
 import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
 import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
@@ -35,6 +37,10 @@ import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface
  *******************************************************************************/
 public class InsertAction
 {
+   private static final Logger LOG = LogManager.getLogger(InsertAction.class);
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -49,4 +55,5 @@ public class InsertAction
       // todo post-customization - can do whatever w/ the result if you want
       return insertResult;
    }
+
 }

src/main/java/com/kingsrook/qqq/backend/core/actions/RunFunctionAction.java

@@ -39,6 +39,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
@@ -47,6 +49,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  *******************************************************************************/
 public class RunFunctionAction
 {
+   private static final Logger LOG = LogManager.getLogger(RunFunctionAction.class);
 
    /*******************************************************************************
     **
@@ -99,11 +102,18 @@ public class RunFunctionAction
       {
          Serializable value = runFunctionRequest.getValue(field.getName());
          if(value == null)
+         {
+            if(field.getDefaultValue() != null)
+            {
+               runFunctionRequest.addValue(field.getName(), field.getDefaultValue());
+            }
+            else
             {
                // todo - check if required?
                fieldsToGet.add(field);
             }
          }
+      }
 
       if(!fieldsToGet.isEmpty())
       {
@@ -183,7 +193,7 @@ public class RunFunctionAction
       {
          runFunctionResult = new RunFunctionResult();
          runFunctionResult.setError("Error running function code: " + e.getMessage());
-         e.printStackTrace();
+         LOG.info("Error running function code", e);
       }
 
       return (runFunctionResult);

src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java

@@ -73,6 +73,8 @@ public class CsvToQRecordAdapter
                   .withTrim());
 
             List<String> headers = csvParser.getHeaderNames();
+            headers = makeHeadersUnique(headers);
+
             List<CSVRecord> csvRecords = csvParser.getRecords();
             for(CSVRecord csvRecord : csvRecords)
             {
@@ -80,9 +82,9 @@ public class CsvToQRecordAdapter
                // put values from the CSV record into a map of header -> value //
                //////////////////////////////////////////////////////////////////
                Map<String, String> csvValues = new HashMap<>();
-               for(String header : headers)
+               for(int i=0; i<headers.size(); i++)
                {
-                  csvValues.put(header, csvRecord.get(header));
+                  csvValues.put(headers.get(i), csvRecord.get(i));
                }
 
                //////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -144,4 +146,41 @@ public class CsvToQRecordAdapter
       return (rs);
    }
 
+
+
+   /*******************************************************************************
+    ** For a list of headers, if any duplicates are found, add a numeric suffix
+    ** to the duplicates.
+    **
+    ** So this header row:  A,B,C,C,C
+    ** Would become:  A,B,C,C 2,C 3
+    **
+    ** See unit test for more scenarios - some of which we do not handle well yet,
+    ** such as "C 2, C, C 3"
+    *******************************************************************************/
+   protected List<String> makeHeadersUnique(List<String> headers)
+   {
+      Map<String, Integer> countsByHeader = new HashMap<>();
+      List<String>         rs             = new ArrayList<>();
+
+      for(String header : headers)
+      {
+         String headerToUse = header;
+         String headerWithoutSuffix = header.replaceFirst(" \\d+$", "");
+
+         if(countsByHeader.containsKey(headerWithoutSuffix))
+         {
+            int suffix = countsByHeader.get(headerWithoutSuffix) + 1;
+            countsByHeader.put(headerWithoutSuffix, suffix);
+            headerToUse = headerWithoutSuffix + " " + suffix;
+         }
+         else
+         {
+            countsByHeader.put(headerWithoutSuffix, 1);
+         }
+         rs.add(headerToUse);
+      }
+      return (rs);
+   }
+
 }

src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapter.java

@@ -55,6 +55,7 @@ public class JsonToQFieldMappingAdapter
       try
       {
          JSONObject jsonObject = JsonUtils.toJSONObject(json);
+         jsonObject = promoteInnerMappingIfAppropriate(jsonObject);
 
          //////////////////////////////////////////////////////////////////////////////////////////////
          // look at the keys in the mapping - if they're strings, then we're doing key-based mapping //
@@ -101,6 +102,29 @@ public class JsonToQFieldMappingAdapter
 
 
 
+   /*******************************************************************************
+    ** So - this class was first written assuming that the JSON it would take would
+    ** just be a mapping - e.g., {a:b, c:d} or {a:0, b:1}.
+    **
+    ** But - it turns out, callers may expect that they can create an instance of
+    ** AbstractQFieldMapping, then serialize it, then de-serialize it, and that seems sane.
+    **
+    ** So - this method tries to determine if the JSON Object we took in looks like
+    ** a serialized from of a AbstractQFieldMapping - and if so, then it "promotes"
+    ** the "mapping" object from within that outer json object, since the rest of
+    ** this class knows how to (and expects to) handle that object.
+    *******************************************************************************/
+   private JSONObject promoteInnerMappingIfAppropriate(JSONObject jsonObject)
+   {
+      if(jsonObject.has("mapping") && jsonObject.has("sourceType") && jsonObject.keySet().size() == 2)
+      {
+         return (jsonObject.getJSONObject("mapping"));
+      }
+      return (jsonObject);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.instances;
 
 
 import java.util.Locale;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
@@ -33,7 +34,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 /*******************************************************************************
  ** As part of helping a QInstance be created and/or validated, apply some default
- ** transfomations to it, such as populating missing labels based on names.
+ ** transformations to it, such as populating missing labels based on names.
  **
  *******************************************************************************/
 public class QInstanceEnricher
@@ -52,6 +53,21 @@ public class QInstanceEnricher
       {
          qInstance.getProcesses().values().forEach(this::enrich);
       }
+
+      if(qInstance.getBackends() != null)
+      {
+         qInstance.getBackends().values().forEach(this::enrich);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void enrich(QBackendMetaData qBackendMetaData)
+   {
+      qBackendMetaData.enrich();
    }
 
 

src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -67,7 +67,6 @@ public class QInstanceValidator
       }
       catch(Exception e)
       {
-         e.printStackTrace();
          throw (new QInstanceValidationException("Error enriching qInstance prior to validation.", e));
       }
 
@@ -134,7 +133,6 @@ public class QInstanceValidator
       }
       catch(Exception e)
       {
-         e.printStackTrace();
          throw (new QInstanceValidationException("Error performing qInstance validation.", e));
       }
 

src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQRequest.java

@@ -27,6 +27,8 @@ import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
 import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
@@ -35,6 +37,8 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
  *******************************************************************************/
 public abstract class AbstractQRequest
 {
+   private static final Logger LOG = LogManager.getLogger(AbstractQRequest.class);
+
    protected QInstance instance;
    protected QSession session;
 
@@ -68,7 +72,7 @@ public abstract class AbstractQRequest
          }
          catch(QInstanceValidationException e)
          {
-            System.err.println(e.getMessage());
+            LOG.warn(e);
             throw (new IllegalArgumentException("QInstance failed validation" + e.getMessage()));
          }
       }

src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunFunctionResult.java

@@ -40,6 +40,20 @@ public class RunFunctionResult extends AbstractQResult
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String toString()
+   {
+      return "RunFunctionResult{error='" + error
+         + ",records.size()=" + (processState == null ? null : processState.getRecords().size())
+         + ",values=" + (processState == null ? null : processState.getValues())
+         + "}";
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java

@@ -146,4 +146,16 @@ public class QBackendMetaData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Called by the QInstanceEnricher - to do backend-type-specific enrichments.
+    ** Original use case is:  reading secrets into fields (e.g., passwords).
+    *******************************************************************************/
+   public void enrich()
+   {
+      ////////////////////////
+      // noop in base class //
+      ////////////////////////
+   }
 }

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QCodeUsage.java

@@ -28,5 +28,6 @@ package com.kingsrook.qqq.backend.core.model.metadata;
  *******************************************************************************/
 public enum QCodeUsage
 {
-   FUNCTION
+   FUNCTION, // a step in a process
+   CUSTOMIZER // a function to customize part of a QQQ table's behavior
 }

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QFieldMetaData.java

@@ -22,6 +22,9 @@
 package com.kingsrook.qqq.backend.core.model.metadata;
 
 
+import java.io.Serializable;
+
+
 /*******************************************************************************
  ** Meta-data to represent a single field in a table.
  **
@@ -33,6 +36,7 @@ public class QFieldMetaData
    private String backendName;
    private QFieldType type;
 
+   private Serializable defaultValue;
    private String possibleValueSourceName;
 
 
@@ -216,4 +220,37 @@ public class QFieldMetaData
       return (this);
    }
 
+   /*******************************************************************************
+    ** Getter for defaultValue
+    **
+    *******************************************************************************/
+   public Serializable getDefaultValue()
+   {
+      return defaultValue;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for defaultValue
+    **
+    *******************************************************************************/
+   public void setDefaultValue(Serializable defaultValue)
+   {
+      this.defaultValue = defaultValue;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QFieldMetaData withDefaultValue(Serializable defaultValue)
+   {
+      this.defaultValue = defaultValue;
+      return (this);
+   }
+
+
+
 }

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QSecretReader.java

@@ -0,0 +1,89 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata;
+
+
+import java.util.Map;
+
+
+/*******************************************************************************
+ ** To avoid having secrets (passwords, access keys, etc) committed into meta data
+ ** files, this class is used by the Enricher to "promote" values, such as ${env.ACCESS_KEY}
+ ** to be read from the environment (or other secret providers (to be implemented)).
+ *******************************************************************************/
+public class QSecretReader
+{
+   private Map<String, String> customEnvironment;
+
+
+
+   /*******************************************************************************
+    ** Translate a secret.
+    **
+    ** If input is null, output is null.
+    ** If input looks like ${env.X}, then the return value is the value of the env variable 'X'
+    ** Else the output is the input.
+    *******************************************************************************/
+   public String readSecret(String value)
+   {
+      if(value == null)
+      {
+         return (null);
+      }
+
+      if(value.startsWith("${env.") && value.endsWith("}"))
+      {
+         String envVarName = value.substring(6).replaceFirst("}$", "");
+         String envValue   = getEnvironment().get(envVarName);
+         return (envValue);
+      }
+
+      return (value);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for customEnvironment - protected - meant to be called (at least at this
+    ** time), only in unit test
+    **
+    *******************************************************************************/
+   protected void setCustomEnvironment(Map<String, String> customEnvironment)
+   {
+      this.customEnvironment = customEnvironment;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Map<String, String> getEnvironment()
+   {
+      if(this.customEnvironment != null)
+      {
+         return (this.customEnvironment);
+      }
+
+      return System.getenv();
+   }
+}

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QTableMetaData.java

@@ -23,8 +23,11 @@ package com.kingsrook.qqq.backend.core.model.metadata;
 
 
 import java.io.Serializable;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 
 /*******************************************************************************
@@ -51,6 +54,8 @@ public class QTableMetaData implements Serializable
 
    private QTableBackendDetails backendDetails;
 
+   private Map<String, QCodeReference> customizers;
+
 
 
    /*******************************************************************************
@@ -241,6 +246,21 @@ public class QTableMetaData implements Serializable
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QTableMetaData withFields(List<QFieldMetaData> fields)
+   {
+      this.fields = new LinkedHashMap<>();
+      for(QFieldMetaData field : fields)
+      {
+         this.addField(field);
+      }
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -250,6 +270,12 @@ public class QTableMetaData implements Serializable
       {
          this.fields = new LinkedHashMap<>();
       }
+
+      if(this.fields.containsKey(field.getName()))
+      {
+         throw (new IllegalArgumentException("Attempt to add a second field with name [" + field.getName() + "] to table [" + name + "]."));
+      }
+
       this.fields.put(field.getName(), field);
    }
 
@@ -291,6 +317,7 @@ public class QTableMetaData implements Serializable
    }
 
 
+
    /*******************************************************************************
     ** Fluent Setter for backendDetails
     **
@@ -301,4 +328,72 @@ public class QTableMetaData implements Serializable
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Optional<QCodeReference> getCustomizer(String customizerName)
+   {
+      if(customizers == null)
+      {
+         return (Optional.empty());
+      }
+
+      QCodeReference function = customizers.get(customizerName);
+      if(function == null)
+      {
+         throw (new IllegalArgumentException("Customizer  [" + customizerName + "] was not found in table [" + name + "]."));
+      }
+
+      return (Optional.of(function));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Map<String, QCodeReference> getCustomizers()
+   {
+      return customizers;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for customizers
+    **
+    *******************************************************************************/
+   public void setCustomizers(Map<String, QCodeReference> customizers)
+   {
+      this.customizers = customizers;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QTableMetaData withCustomizer(String role, QCodeReference customizer)
+   {
+      if(this.customizers == null)
+      {
+         this.customizers = new HashMap<>();
+      }
+      // todo - check for dupes?
+      this.customizers.put(role, customizer);
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QTableMetaData withCustomizers(Map<String, QCodeReference> customizers)
+   {
+      this.customizers = customizers;
+      return (this);
+   }
+
 }

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionInputMetaData.java

@@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
 
 
@@ -72,6 +74,33 @@ public class QFunctionInputMetaData
 
 
 
+   /*******************************************************************************
+    ** Getter a field with the given name
+    **
+    *******************************************************************************/
+   public Optional<QFieldMetaData> getField(String name)
+   {
+      return (fieldList.stream().filter(field -> name.equals(field.getName())).findFirst());
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter a field with the given name - throwing if it wasn't found
+    **
+    *******************************************************************************/
+   public QFieldMetaData getFieldThrowing(String name) throws QException
+   {
+      Optional<QFieldMetaData> field = fieldList.stream().filter(f -> name.equals(f.getName())).findFirst();
+      if(field.isEmpty())
+      {
+         throw (new QException("Could not find field [" + name + "] in function input meta data"));
+      }
+      return (field.get());
+   }
+
+
+
    /*******************************************************************************
     ** Getter for fieldList
     **

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/QFieldMappingDeserializer.java

@@ -0,0 +1,69 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.serialization;
+
+
+import java.io.IOException;
+import com.fasterxml.jackson.core.JacksonException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.TreeNode;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.node.NullNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
+import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
+
+
+/*******************************************************************************
+ ** Jackson custom deserialization class, to return an appropriate sub-type of
+ ** A QBackendMetaData, based on the backendType specified within.
+ *******************************************************************************/
+public class QFieldMappingDeserializer extends JsonDeserializer<AbstractQFieldMapping>
+{
+   @Override
+   public AbstractQFieldMapping deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JacksonException
+   {
+      TreeNode treeNode = jsonParser.readValueAsTree();
+
+      TreeNode sourceTypeTreeNode = treeNode.get("sourceType");
+      if(sourceTypeTreeNode == null || sourceTypeTreeNode instanceof NullNode)
+      {
+         throw new IOException("Missing sourceType in serializedMapping");
+      }
+
+      if(!(sourceTypeTreeNode instanceof TextNode textNode))
+      {
+         throw new IOException("sourceType is not a string value (is: " + sourceTypeTreeNode.getClass().getSimpleName() + ")");
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      // get the value of the backendType json node, and use it to look up the qBackendModule object //
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      String backendType = textNode.asText();
+
+      QBackendModuleInterface backendModule = DeserializerUtils.getBackendModule(treeNode);
+
+      return null;
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/core/modules/QBackendModuleDispatcher.java

@@ -42,7 +42,7 @@ public class QBackendModuleDispatcher
 {
    private static final Logger LOG = LogManager.getLogger(QBackendModuleDispatcher.class);
 
-   private Map<String, String> backendTypeToModuleClassNameMap;
+   private static Map<String, String> backendTypeToModuleClassNameMap;
 
 
 
@@ -51,6 +51,21 @@ public class QBackendModuleDispatcher
     *******************************************************************************/
    public QBackendModuleDispatcher()
    {
+      initBackendTypeToModuleClassNameMap();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void initBackendTypeToModuleClassNameMap()
+   {
+      if(backendTypeToModuleClassNameMap != null)
+      {
+         return;
+      }
+
       backendTypeToModuleClassNameMap = new HashMap<>();
 
       String[] moduleClassNames = new String[]
@@ -80,6 +95,22 @@ public class QBackendModuleDispatcher
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void registerBackendModule(QBackendModuleInterface moduleInstance)
+   {
+      initBackendTypeToModuleClassNameMap();
+      String backendType = moduleInstance.getBackendType();
+      if(backendTypeToModuleClassNameMap.containsKey(backendType))
+      {
+         LOG.info("Overwriting backend type [" + backendType + "] with [" + moduleInstance.getClass() + "]");
+      }
+      backendTypeToModuleClassNameMap.put(backendType, moduleInstance.getClass().getName());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

src/main/java/com/kingsrook/qqq/backend/core/modules/mock/MockQueryAction.java

@@ -71,7 +71,6 @@ public class MockQueryAction implements QueryInterface
       }
       catch(Exception e)
       {
-         e.printStackTrace();
          throw new QException("Error executing query", e);
       }
    }

src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java

@@ -39,8 +39,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 public class BasicETLProcess
 {
    public static final String PROCESS_NAME            = "etl.basic";
+   public static final String FUNCTION_NAME_EXTRACT   = "extract";
+   public static final String FUNCTION_NAME_TRANSFORM = "transform";
+   public static final String FUNCTION_NAME_LOAD      = "load";
    public static final String FIELD_SOURCE_TABLE      = "sourceTable";
    public static final String FIELD_DESTINATION_TABLE = "destinationTable";
+   public static final String FIELD_MAPPING_JSON      = "mappingJSON";
    public static final String FIELD_RECORD_COUNT      = "recordCount";
 
 
@@ -51,7 +55,7 @@ public class BasicETLProcess
    public QProcessMetaData defineProcessMetaData()
    {
       QFunctionMetaData extractFunction = new QFunctionMetaData()
-         .withName("extract")
+         .withName(FUNCTION_NAME_EXTRACT)
          .withCode(new QCodeReference()
             .withName(BasicETLExtractFunction.class.getName())
             .withCodeType(QCodeType.JAVA)
@@ -59,8 +63,18 @@ public class BasicETLProcess
          .withInputData(new QFunctionInputMetaData()
             .addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)));
 
+      QFunctionMetaData transformFunction = new QFunctionMetaData()
+         .withName(FUNCTION_NAME_TRANSFORM)
+         .withCode(new QCodeReference()
+            .withName(BasicETLTransformFunction.class.getName())
+            .withCodeType(QCodeType.JAVA)
+            .withCodeUsage(QCodeUsage.FUNCTION))
+         .withInputData(new QFunctionInputMetaData()
+            .addField(new QFieldMetaData(FIELD_MAPPING_JSON, QFieldType.STRING))
+            .addField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)));
+
       QFunctionMetaData loadFunction = new QFunctionMetaData()
-         .withName("load")
+         .withName(FUNCTION_NAME_LOAD)
          .withCode(new QCodeReference()
             .withName(BasicETLLoadFunction.class.getName())
             .withCodeType(QCodeType.JAVA)
@@ -73,6 +87,7 @@ public class BasicETLProcess
       return new QProcessMetaData()
          .withName(PROCESS_NAME)
          .addFunction(extractFunction)
+         .addFunction(transformFunction)
          .addFunction(loadFunction);
    }
 }

src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunction.java

@@ -0,0 +1,99 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.adapters.JsonToQFieldMappingAdapter;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
+import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
+import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Function body for performing the Extract step of a basic ETL process.
+ *******************************************************************************/
+public class BasicETLTransformFunction implements FunctionBody
+{
+   @Override
+   public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
+   {
+      ////////////////////////////////////////////////////////////////////////////////////////////
+      // exit early with no-op if no records made it here, or if we don't have a mapping to use //
+      ////////////////////////////////////////////////////////////////////////////////////////////
+      if(CollectionUtils.nullSafeIsEmpty(runFunctionRequest.getRecords()))
+      {
+         return;
+      }
+
+      String mappingJSON = runFunctionRequest.getValueString(BasicETLProcess.FIELD_MAPPING_JSON);
+      if(!StringUtils.hasContent(mappingJSON))
+      {
+         return;
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // require that the mapping be a key-based mapping (can't use indexes into qRecord values) //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      AbstractQFieldMapping<?> mapping = new JsonToQFieldMappingAdapter().buildMappingFromJson(mappingJSON);
+      if(!(mapping instanceof QKeyBasedFieldMapping keyBasedFieldMapping))
+      {
+         throw (new QException("Mapping was not a Key-based mapping type.  Was a : " + mapping.getClass().getName()));
+      }
+
+      String         tableName     = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
+      QTableMetaData table         = runFunctionRequest.getInstance().getTable(tableName);
+      List<QRecord>  mappedRecords = applyMapping(runFunctionRequest.getRecords(), table, keyBasedFieldMapping);
+      runFunctionResult.setRecords(mappedRecords);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<QRecord> applyMapping(List<QRecord> input, QTableMetaData table, QKeyBasedFieldMapping mapping)
+   {
+      List<QRecord> output = new ArrayList<>();
+      for(QRecord inputRecord : input)
+      {
+         QRecord outputRecord = new QRecord();
+         output.add(outputRecord);
+         for(QFieldMetaData field : table.getFields().values())
+         {
+            String fieldSource = mapping.getFieldSource(field.getName());
+            outputRecord.setValue(field.getName(), inputRecord.getValue(fieldSource));
+         }
+      }
+      return (output);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java

@@ -29,6 +29,8 @@ import java.io.Serializable;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
@@ -36,6 +38,8 @@ import org.apache.commons.io.FileUtils;
  *******************************************************************************/
 public class TempFileStateProvider implements StateProviderInterface
 {
+   private static final Logger LOG = LogManager.getLogger(TempFileStateProvider.class);
+
    private static TempFileStateProvider instance;
 
 
@@ -76,8 +80,8 @@ public class TempFileStateProvider implements StateProviderInterface
       }
       catch(IOException e)
       {
-         // todo better
+         LOG.error("Error putting state into file", e);
-         e.printStackTrace();
+         throw (new RuntimeException("Error storing state", e));
       }
    }
 
@@ -98,9 +102,10 @@ public class TempFileStateProvider implements StateProviderInterface
       {
          return (Optional.empty());
       }
-      catch(IOException ie)
+      catch(IOException e)
       {
-         throw new RuntimeException("Error loading state from file", ie);
+         LOG.error("Error getting state from file", e);
+         throw (new RuntimeException("Error retreiving state", e));
       }
    }
 

src/main/resources/log4j2.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration>
+   <Properties>
+      <Property name="LOG_PATTERN">%highlight{%d{ISO8601} | %-6r | %-5p | %markerSimpleName | %c{1} | %t{1} | %m%n}{FATAL=red, ERROR=red, WARN=blue, INFO=green, METRICS=magenta, DEBUG=cyan, TRACE=white}</Property>
+   </Properties>
+   <CustomLevels>
+      <CustomLevel name="METRICS" intLevel="450"/>
+   </CustomLevels>
+   <Appenders>
+      <File name="FullLogFile" fileName="log/full_log.log">
+         <LevelRangeFilter minLevel="ERROR" maxLevel="all" onMatch="ACCEPT" onMismatch="DENY"/>
+         <PatternLayout pattern="${LOG_PATTERN}"/>
+      </File>
+      <Console name="STDOUT" target="SYSTEM_OUT">
+         <LevelRangeFilter minLevel="ERROR" maxLevel="METRICS" onMatch="ACCEPT" onMismatch="DENY"/>
+         <PatternLayout pattern="${LOG_PATTERN}"/>
+      </Console>
+   </Appenders>
+   <Loggers>
+      <Logger name="org.apache.log4j.xml" additivity="false">
+      </Logger>
+      <Root level="all">
+         <AppenderRef ref="STDOUT"/>
+         <!-- Uncomment only if you need the full_log.
+         <AppenderRef ref="FullLogFile" />
+         -->
+      </Root>
+   </Loggers>
+</Configuration>

src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java

@@ -27,6 +27,7 @@ import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFi
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -230,4 +231,54 @@ class CsvToQRecordAdapterTest
       assertEquals("1981-01-01", qRecord2.getValue("birthDate"));
    }
 
+
+
+   /*******************************************************************************
+    ** In this test - we've got CSV data with duplicated header names.
+    ** In our mapping, we're seeing the suffixes of " 2" and " 3" addd to those
+    ** header names on the RHS.
+    *******************************************************************************/
+   @Test
+   public void test_duplicatedColumnHeaders()
+   {
+      QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping()
+         .withMapping("id", "id")
+         .withMapping("createDate", "date")
+         .withMapping("modifyDate", "date 2")
+         .withMapping("firstName", "name")
+         .withMapping("lastName", "name 2")
+         .withMapping("birthDate", "date 3")
+         .withMapping("email", "email");
+
+      CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+      List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv("""
+         id,date,date,name,name,date,email
+         1,2022-06-26,2022-06-26,John,Doe,1980-01-01,john@kingsrook.com
+         """, TestUtils.defineTablePerson(), mapping);
+      assertNotNull(qRecords);
+      assertEquals(1, qRecords.size());
+      QRecord qRecord1 = qRecords.get(0);
+      assertEquals("John", qRecord1.getValue("firstName"));
+      assertEquals("Doe", qRecord1.getValue("lastName"));
+      assertEquals("1980-01-01", qRecord1.getValue("birthDate"));
+      assertEquals("2022-06-26", qRecord1.getValue("createDate"));
+      assertEquals("2022-06-26", qRecord1.getValue("modifyDate"));
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testMakeHeadersUnique()
+   {
+      CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+      Assertions.assertEquals(List.of("A", "B", "C"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C")));
+      Assertions.assertEquals(List.of("A", "B", "C", "C 2", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C", "C", "C")));
+      Assertions.assertEquals(List.of("C", "A", "C 2", "B", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("C", "A", "C", "B", "C")));
+      Assertions.assertEquals(List.of("A", "B", "C", "C 2", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C", "C 2", "C")));
+      Assertions.assertEquals(List.of("A", "B", "C", "C 2", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C", "C 2", "C 3")));
+      // todo - this is what the method header comment means when it says we don't handle all cases well...
+      //  Assertions.assertEquals(List.of("A", "B", "C", "C 2", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C 2", "C", "C 3")));
+   }
 }

src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapterTest.java

@@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.adapters;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFieldMapping;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -130,6 +132,38 @@ class JsonToQFieldMappingAdapterTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_deserializeSerializedQKeyBasedFieldMapping()
+   {
+      QIndexBasedFieldMapping original = new QIndexBasedFieldMapping()
+         .withMapping("foo", 0)
+         .withMapping("bar", 1);
+      String json = JsonUtils.toJson(original);
+      AbstractQFieldMapping<?> deserialized = new JsonToQFieldMappingAdapter().buildMappingFromJson(json);
+      Assertions.assertThat(deserialized).usingRecursiveComparison().isEqualTo(original);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_deserializeSerializedQIndexBasedFieldMapping()
+   {
+      QKeyBasedFieldMapping original = new QKeyBasedFieldMapping()
+         .withMapping("foo", "Fu")
+         .withMapping("bar", "Bahr");
+      String json = JsonUtils.toJson(original);
+      AbstractQFieldMapping<?> deserialized = new JsonToQFieldMappingAdapter().buildMappingFromJson(json);
+      Assertions.assertThat(deserialized).usingRecursiveComparison().isEqualTo(original);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -140,6 +174,7 @@ class JsonToQFieldMappingAdapterTest
    }
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/

src/test/java/com/kingsrook/qqq/backend/core/model/metadata/QSecretReaderTest.java

@@ -0,0 +1,55 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata;
+
+
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+
+/*******************************************************************************
+ ** Unit test for QSecretReader
+ *******************************************************************************/
+class QSecretReaderTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testReadSecret()
+   {
+      QSecretReader secretReader = new QSecretReader();
+      String        key          = "CUSTOM_PROPERTY";
+      String        value        = "ABCD-9876";
+      secretReader.setCustomEnvironment(Map.of(key, value));
+
+      assertNull(secretReader.readSecret(null));
+      assertEquals("foo", secretReader.readSecret("foo"));
+      assertNull(secretReader.readSecret("${env.NOT-" + key + "}"));
+      assertEquals(value, secretReader.readSecret("${env." + key + "}"));
+      assertEquals("${env.NOT-" + key, secretReader.readSecret("${env.NOT-" + key));
+   }
+
+}

src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java

@@ -26,6 +26,8 @@ import com.kingsrook.qqq.backend.core.actions.RunProcessAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessRequest;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessResult;
+import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -40,7 +42,7 @@ class BasicETLProcessTest
 {
 
    /*******************************************************************************
-    **
+    ** Simplest happy path
     *******************************************************************************/
    @Test
    public void test() throws QException
@@ -50,6 +52,7 @@ class BasicETLProcessTest
       request.setProcessName(BasicETLProcess.PROCESS_NAME);
       request.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.defineTablePerson().getName());
       request.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.definePersonFileTable().getName());
+      request.addValue(BasicETLProcess.FIELD_MAPPING_JSON, "");
 
       RunProcessResult result = new RunProcessAction().execute(request);
       assertNotNull(result);
@@ -58,4 +61,30 @@ class BasicETLProcessTest
    }
 
 
+
+   /*******************************************************************************
+    ** Basic example of doing a mapping transformation
+    *******************************************************************************/
+   @Test
+   public void testMappingTransformation() throws QException
+   {
+      RunProcessRequest request = new RunProcessRequest(TestUtils.defineInstance());
+      request.setSession(TestUtils.getMockSession());
+      request.setProcessName(BasicETLProcess.PROCESS_NAME);
+      request.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.definePersonFileTable().getName());
+      request.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.defineTableIdAndNameOnly().getName());
+
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // define our mapping from destination-table field names to source-table field names //
+      ///////////////////////////////////////////////////////////////////////////////////////
+      QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping().withMapping("name", "firstName");
+      // request.addValue(BasicETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping.getMapping()));
+      request.addValue(BasicETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping));
+
+      RunProcessResult result = new RunProcessAction().execute(request);
+      assertNotNull(result);
+      assertNull(result.getError());
+      assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process");
+   }
+
 }

src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -68,6 +68,7 @@ public class TestUtils
       qInstance.addBackend(defineBackend());
       qInstance.addTable(defineTablePerson());
       qInstance.addTable(definePersonFileTable());
+      qInstance.addTable(defineTableIdAndNameOnly());
       qInstance.addPossibleValueSource(defineStatesPossibleValueSource());
       qInstance.addProcess(defineProcessGreetPeople());
       qInstance.addProcess(defineProcessAddToPeoplesAge());
@@ -153,6 +154,22 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    ** Define simple table with just an id and name
+    *******************************************************************************/
+   public static QTableMetaData defineTableIdAndNameOnly()
+   {
+      return new QTableMetaData()
+         .withName("idAndNameOnly")
+         .withLabel("Id and Name Only")
+         .withBackendName(DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("name", QFieldType.STRING));
+   }
+
+
+
    /*******************************************************************************
     ** Define the 'greet people' process
     *******************************************************************************/