Checkpoint - WIP on MetaData action, Insert action, csv/json record adapters

pom.xml

@@ -41,6 +41,11 @@
          <artifactId>json</artifactId>
          <version>20210307</version>
       </dependency>
+      <dependency>
+         <groupId>org.apache.commons</groupId>
+         <artifactId>commons-csv</artifactId>
+         <version>1.8</version>
+      </dependency>
 
       <!-- Common deps for all qqq modules -->
       <dependency>
@@ -122,6 +127,14 @@
    </build>
 
 <!--
+   <distributionManagement>
+      <snapshotRepository>
+         <id>nexus-snapshots</id>
+         <url>http://localhost:8088/repository/maven-snapshots</url>
+      </snapshotRepository>
+   </distributionManagement>
+-->
+
    <distributionManagement>
       <repository>
         <id>github</id>
@@ -129,6 +142,5 @@
         <url>https://maven.pkg.github.com/Kingsrook/qqq-maven-registry</url>
       </repository>
    </distributionManagement>
--->
 
 </project>

src/main/java/com/kingsrook/qqq/backend/core/actions/MetaDataAction.java

@@ -0,0 +1,37 @@
+package com.kingsrook.qqq.backend.core.actions;
+
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.MetaDataRequest;
+import com.kingsrook.qqq.backend.core.model.actions.MetaDataResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class MetaDataAction
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public MetaDataResult execute(MetaDataRequest metaDataRequest) throws QException
+   {
+      // todo pre-customization - just get to modify the request?
+      MetaDataResult metaDataResult = new MetaDataResult();
+
+      Map<String, QFrontendTableMetaData> tables = new LinkedHashMap<>();
+      for(Map.Entry<String, QTableMetaData> entry : metaDataRequest.getInstance().getTables().entrySet())
+      {
+         tables.put(entry.getKey(), new QFrontendTableMetaData(entry.getValue(), false));
+      }
+
+      metaDataResult.setTables(tables);
+      // todo post-customization - can do whatever w/ the result if you want
+
+      return metaDataResult;
+   }
+}

src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java

@@ -0,0 +1,122 @@
+package com.kingsrook.qqq.backend.core.adapters;
+
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractQFieldMapping;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class CsvToQRecordAdapter
+{
+
+   /*******************************************************************************
+    ** todo - meta-data validation, mapping, type handling
+    *******************************************************************************/
+   public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping)
+   {
+      if(!StringUtils.hasContent(csv))
+      {
+         throw (new IllegalArgumentException("Empty csv value was provided."));
+      }
+
+      List<QRecord> rs = new ArrayList<>();
+      try
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if there's no mapping (e.g., table-standard field names), or key-based mapping, then first row is headers //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(mapping == null || AbstractQFieldMapping.SourceType.KEY.equals(mapping.getSourceType()))
+         {
+            CSVParser csvParser = new CSVParser(new StringReader(csv),
+               CSVFormat.DEFAULT
+                  .withFirstRecordAsHeader()
+                  .withIgnoreHeaderCase()
+                  .withTrim());
+
+            List<String> headers = csvParser.getHeaderNames();
+            List<CSVRecord> csvRecords = csvParser.getRecords();
+            for(CSVRecord csvRecord : csvRecords)
+            {
+               //////////////////////////////////////////////////////////////////
+               // put values from the CSV record into a map of header -> value //
+               //////////////////////////////////////////////////////////////////
+               Map<String, String> csvValues = new HashMap<>();
+               for(String header : headers)
+               {
+                  csvValues.put(header, csvRecord.get(header));
+               }
+
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////
+               QRecord qRecord = new QRecord();
+               rs.add(qRecord);
+               for(QFieldMetaData field : table.getFields().values())
+               {
+                  String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
+                  qRecord.setValue(field.getName(), csvValues.get(fieldSource));
+               }
+            }
+         }
+         else if(AbstractQFieldMapping.SourceType.INDEX.equals(mapping.getSourceType()))
+         {
+            ///////////////////////////////
+            // else, index-based mapping //
+            ///////////////////////////////
+            CSVParser csvParser = new CSVParser(new StringReader(csv),
+               CSVFormat.DEFAULT
+                  .withTrim());
+
+            List<CSVRecord> csvRecords = csvParser.getRecords();
+            for(CSVRecord csvRecord : csvRecords)
+            {
+               /////////////////////////////////////////////////////////////////
+               // put values from the CSV record into a map of index -> value //
+               /////////////////////////////////////////////////////////////////
+               Map<Integer, String> csvValues = new HashMap<>();
+               int index = 1;
+               for(String value : csvRecord)
+               {
+                  csvValues.put(index++, value);
+               }
+
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////
+               QRecord qRecord = new QRecord();
+               rs.add(qRecord);
+               for(QFieldMetaData field : table.getFields().values())
+               {
+                  Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName());
+                  qRecord.setValue(field.getName(), csvValues.get(fieldIndex));
+               }
+            }
+         }
+         else
+         {
+            throw (new IllegalArgumentException("Unrecognized mapping source type: " + mapping.getSourceType()));
+         }
+      }
+      catch(IOException e)
+      {
+         throw (new IllegalArgumentException("Error parsing CSV: " + e.getMessage(), e));
+      }
+
+      return (rs);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapter.java

@@ -2,6 +2,7 @@ package com.kingsrook.qqq.backend.core.adapters;
 
 
 import com.kingsrook.qqq.backend.core.model.actions.AbstractQFieldMapping;
+import com.kingsrook.qqq.backend.core.model.actions.QIndexBasedFieldMapping;
 import com.kingsrook.qqq.backend.core.model.actions.QKeyBasedFieldMapping;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -33,12 +34,37 @@ public class JsonToQFieldMappingAdapter
          // look at the keys in the mapping - if they're strings, then we're doing key-based mapping //
          // if they're numbers, then we're doing index based -- and if they're a mix, that's illegal //
          //////////////////////////////////////////////////////////////////////////////////////////////
+         AbstractQFieldMapping.SourceType sourceType = determineSourceType(jsonObject);
 
-         QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping();
+         @SuppressWarnings("rawtypes")
-         for(String key : jsonObject.keySet())
+         AbstractQFieldMapping mapping = null;
+
+         switch(sourceType)
          {
-            mapping.addMapping(key, jsonObject.getString(key));
+            case KEY:
+            {
+               mapping = new QKeyBasedFieldMapping();
+               for(String fieldName : jsonObject.keySet())
+               {
+                  ((QKeyBasedFieldMapping) mapping).addMapping(fieldName, jsonObject.getString(fieldName));
+               }
+               break;
+            }
+            case INDEX:
+            {
+               mapping = new QIndexBasedFieldMapping();
+               for(String fieldName : jsonObject.keySet())
+               {
+                  ((QIndexBasedFieldMapping) mapping).addMapping(fieldName, jsonObject.getInt(fieldName));
+               }
+               break;
+            }
+            default:
+            {
+               throw (new IllegalArgumentException("Unsupported sourceType: " + sourceType));
+            }
          }
+
          return (mapping);
       }
       catch(JSONException je)
@@ -47,4 +73,30 @@ public class JsonToQFieldMappingAdapter
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private AbstractQFieldMapping.SourceType determineSourceType(JSONObject jsonObject)
+   {
+      for(String fieldName : jsonObject.keySet())
+      {
+         Object sourceObject = jsonObject.get(fieldName);
+         if(sourceObject instanceof String)
+         {
+            return (AbstractQFieldMapping.SourceType.KEY);
+         }
+         else if(sourceObject instanceof Integer)
+         {
+            return (AbstractQFieldMapping.SourceType.INDEX);
+         }
+         else
+         {
+            throw new IllegalArgumentException("Source object is unsupported type: " + sourceObject.getClass().getSimpleName());
+         }
+      }
+      throw new IllegalArgumentException("No fields were found in the mapping.");
+   }
+
 }

src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQRecordAdapter.java

@@ -4,7 +4,9 @@ package com.kingsrook.qqq.backend.core.adapters;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractQFieldMapping;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import org.json.JSONArray;
@@ -21,7 +23,7 @@ public class JsonToQRecordAdapter
    /*******************************************************************************
     ** todo - meta-data validation, mapping, type handling
     *******************************************************************************/
-   public List<QRecord> buildRecordsFromJson(String json)
+   public List<QRecord> buildRecordsFromJson(String json, QTableMetaData table, AbstractQFieldMapping<?> mapping)
    {
       if(!StringUtils.hasContent(json))
       {

src/main/java/com/kingsrook/qqq/backend/core/exceptions/QInstanceValidationException.java

@@ -0,0 +1,82 @@
+package com.kingsrook.qqq.backend.core.exceptions;
+
+
+import java.util.Arrays;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class QInstanceValidationException extends QException
+{
+   private List<String> reasons;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QInstanceValidationException(String message)
+   {
+      super(message);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QInstanceValidationException(List<String> reasons)
+   {
+      super(
+         (reasons != null && reasons.size() > 0)
+            ? "Instance validation failed for the following reasons:  " + StringUtils.joinWithCommasAndAnd(reasons)
+            : "Validation failed, but no reasons were provided");
+
+      if(reasons != null && reasons.size() > 0)
+      {
+         this.reasons = reasons;
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QInstanceValidationException(String... reasons)
+   {
+      super(
+         (reasons != null && reasons.length > 0)
+            ? "Instance validation failed for the following reasons:  " + StringUtils.joinWithCommasAndAnd(Arrays.stream(reasons).toList())
+            : "Validation failed, but no reasons were provided");
+
+      if(reasons != null && reasons.length > 0)
+      {
+         this.reasons = Arrays.stream(reasons).toList();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QInstanceValidationException(String message, Throwable cause)
+   {
+      super(message, cause);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for reasons
+    **
+    *******************************************************************************/
+   public List<String> getReasons()
+   {
+      return reasons;
+   }
+}

src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -0,0 +1,73 @@
+package com.kingsrook.qqq.backend.core.instances;
+
+
+import java.util.Locale;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class QInstanceEnricher
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void enrich(QInstance qInstance)
+   {
+      if (qInstance.getTables() != null)
+      {
+         qInstance.getTables().values().forEach(this::enrich);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void enrich(QTableMetaData table)
+   {
+      if(!StringUtils.hasContent(table.getLabel()))
+      {
+         table.setLabel(nameToLabel(table.getName()));
+      }
+
+      if (table.getFields() != null)
+      {
+         table.getFields().values().forEach(this::enrich);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void enrich(QFieldMetaData field)
+   {
+      if(!StringUtils.hasContent(field.getLabel()))
+      {
+         field.setLabel(nameToLabel(field.getName()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String nameToLabel(String name)
+   {
+      if(name == null)
+      {
+         return (null);
+      }
+
+      return (name.substring(0, 1).toUpperCase(Locale.ROOT) + name.substring(1).replaceAll("([A-Z])", " $1"));
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidationKey.java

@@ -0,0 +1,18 @@
+package com.kingsrook.qqq.backend.core.instances;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public final class QInstanceValidationKey
+{
+   /*******************************************************************************
+    ** package-private constructor, so that only this package can create an instance
+    ** of this class, but an instance of this class is required to mark an instance
+    ** as validated, so no one can cheat.
+    *******************************************************************************/
+   QInstanceValidationKey()
+   {
+
+   }
+}

src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -0,0 +1,104 @@
+package com.kingsrook.qqq.backend.core.instances;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class QInstanceValidator
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void validate(QInstance qInstance) throws QInstanceValidationException
+   {
+      if(qInstance.getHasBeenValidated())
+      {
+         //////////////////////////////////////////
+         // don't re-validate if previously done //
+         //////////////////////////////////////////
+         return;
+      }
+
+      try
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////////
+         // before validation, enrich the object (e.g., to fill in values that the user doesn't have to //
+         /////////////////////////////////////////////////////////////////////////////////////////////////
+         new QInstanceEnricher().enrich(qInstance);
+      }
+      catch(Exception e)
+      {
+         e.printStackTrace();
+         throw (new QInstanceValidationException("Error enriching qInstance prior to validation.", e));
+      }
+
+      //////////////////////////////////////////////////////////////////////////
+      // do the validation checks - a good qInstance has all conditions TRUE! //
+      //////////////////////////////////////////////////////////////////////////
+      List<String> errors = new ArrayList<>();
+      try
+      {
+         if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getBackends()), "At least 1 backend must be defined."))
+         {
+            qInstance.getBackends().forEach((backendName, backend) ->
+            {
+               assertCondition(errors, Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + ".");
+            });
+         }
+
+         if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getTables()), "At least 1 table must be defined."))
+         {
+            qInstance.getTables().forEach((tableName, table) ->
+            {
+               assertCondition(errors, Objects.equals(tableName, table.getName()), "Inconsistent naming for table: " + tableName + "/" + table.getName() + ".");
+
+               if(assertCondition(errors, StringUtils.hasContent(table.getBackendName()), "Missing backend name for table " + tableName + "."))
+               {
+                  if(CollectionUtils.nullSafeHasContents(qInstance.getBackends()))
+                  {
+                     assertCondition(errors, qInstance.getBackendForTable(tableName) != null, "Unrecognized backend " + table.getBackendName() + " for table " + tableName + ".");
+                  }
+               }
+            });
+         }
+      }
+      catch(Exception e)
+      {
+         e.printStackTrace();
+         throw (new QInstanceValidationException("Error performing qInstance validation.", e));
+      }
+
+      if(!errors.isEmpty())
+      {
+         throw (new QInstanceValidationException(errors));
+      }
+
+      qInstance.setHasBeenValidated(new QInstanceValidationKey());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private boolean assertCondition(List<String> errors, boolean condition, String message)
+   {
+      if(!condition)
+      {
+         errors.add(message);
+      }
+
+      return (condition);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQFieldMapping.java

@@ -2,12 +2,56 @@ package com.kingsrook.qqq.backend.core.model.actions;
 
 
 /*******************************************************************************
+ ** For bulk-loads, define where a QField comes from in an input data source.
  **
  *******************************************************************************/
 public abstract class AbstractQFieldMapping<T>
 {
+
    /*******************************************************************************
-    ** Returns the fieldName for the input 'key' or 'index'.
+    ** Enum to define the types of sources a mapping may use
     *******************************************************************************/
-   public abstract String getMappedField(T t);
+   @SuppressWarnings("rawtypes")
+   public enum SourceType
+   {
+      KEY(String.class),
+      INDEX(Integer.class);
+
+      private Class sourceClass;
+
+
+
+      /*******************************************************************************
+       ** enum constructor
+       *******************************************************************************/
+      SourceType(Class sourceClass)
+      {
+         this.sourceClass = sourceClass;
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for sourceClass
+       **
+       *******************************************************************************/
+      public Class getSourceClass()
+      {
+         return sourceClass;
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** For a given field, return its source - a key (e.g., from a json object or csv
+    ** with a header row) or an index (for a csv w/o a header)
+    *******************************************************************************/
+   public abstract T getFieldSource(String fieldName);
+
+
+   /*******************************************************************************
+    ** for a mapping instance, get what its source-type is
+    *******************************************************************************/
+   public abstract SourceType getSourceType();
 }

src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQRequest.java

@@ -1,8 +1,9 @@
 package com.kingsrook.qqq.backend.core.model.actions;
 
 
-import com.kingsrook.qqq.backend.core.model.QInstance;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
-import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 
 
 /*******************************************************************************
@@ -15,13 +16,6 @@ public abstract class AbstractQRequest
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public abstract QBackendMetaData getBackend();
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -37,6 +31,23 @@ public abstract class AbstractQRequest
    public AbstractQRequest(QInstance instance)
    {
       this.instance = instance;
+
+      ////////////////////////////////////////////////////////////
+      // if this instance hasn't been validated yet, do so now  //
+      // noting that this will also enrich any missing metaData //
+      ////////////////////////////////////////////////////////////
+      if(! instance.getHasBeenValidated())
+      {
+         try
+         {
+            new QInstanceValidator().validate(instance);
+         }
+         catch(QInstanceValidationException e)
+         {
+            System.err.println(e.getMessage());
+            throw (new IllegalArgumentException("QInstance failed validation" + e.getMessage()));
+         }
+      }
    }
 
 

src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQTableRequest.java

@@ -1,8 +1,8 @@
 package com.kingsrook.qqq.backend.core.model.actions;
 
 
-import com.kingsrook.qqq.backend.core.model.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
 
 
@@ -18,13 +18,13 @@ public abstract class AbstractQTableRequest extends AbstractQRequest
    /*******************************************************************************
     **
     *******************************************************************************/
-   @Override
    public QBackendMetaData getBackend()
    {
       return (instance.getBackendForTable(getTableName()));
    }
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/

src/main/java/com/kingsrook/qqq/backend/core/model/actions/InsertRequest.java

@@ -2,8 +2,8 @@ package com.kingsrook.qqq.backend.core.model.actions;
 
 
 import java.util.List;
-import com.kingsrook.qqq.backend.core.model.QInstance;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 
 
 /*******************************************************************************

src/main/java/com/kingsrook/qqq/backend/core/model/actions/MetaDataRequest.java

@@ -0,0 +1,30 @@
+package com.kingsrook.qqq.backend.core.model.actions;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class MetaDataRequest extends AbstractQRequest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public MetaDataRequest()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public MetaDataRequest(QInstance instance)
+   {
+      super(instance);
+   }
+
+}

src/main/java/com/kingsrook/qqq/backend/core/model/actions/MetaDataResult.java

@@ -0,0 +1,37 @@
+package com.kingsrook.qqq.backend.core.model.actions;
+
+
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData;
+
+
+/*******************************************************************************
+ * Result for a metaData action
+ *
+ *******************************************************************************/
+public class MetaDataResult extends AbstractQResult
+{
+   Map<String, QFrontendTableMetaData> tables;
+
+
+
+   /*******************************************************************************
+    ** Getter for tables
+    **
+    *******************************************************************************/
+   public Map<String, QFrontendTableMetaData> getTables()
+   {
+      return tables;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tables
+    **
+    *******************************************************************************/
+   public void setTables(Map<String, QFrontendTableMetaData> tables)
+   {
+      this.tables = tables;
+   }
+}

src/main/java/com/kingsrook/qqq/backend/core/model/actions/QIndexBasedFieldMapping.java

@@ -10,7 +10,7 @@ import java.util.Map;
  *******************************************************************************/
 public class QIndexBasedFieldMapping extends AbstractQFieldMapping<Integer>
 {
-   private Map<Integer, String> mapping;
+   private Map<String, Integer> mapping;
 
 
 
@@ -18,14 +18,14 @@ public class QIndexBasedFieldMapping extends AbstractQFieldMapping<Integer>
     **
     *******************************************************************************/
    @Override
-   public String getMappedField(Integer key)
+   public Integer getFieldSource(String fieldName)
    {
       if(mapping == null)
       {
          return (null);
       }
 
-      return (mapping.get(key));
+      return (mapping.get(fieldName));
    }
 
 
@@ -33,13 +33,24 @@ public class QIndexBasedFieldMapping extends AbstractQFieldMapping<Integer>
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void addMapping(Integer key, String fieldName)
+   @Override
+   public SourceType getSourceType()
+   {
+      return (SourceType.INDEX);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addMapping(String fieldName, Integer key)
    {
       if(mapping == null)
       {
          mapping = new LinkedHashMap<>();
       }
-      mapping.put(key, fieldName);
+      mapping.put(fieldName, key);
    }
 
 
@@ -47,9 +58,9 @@ public class QIndexBasedFieldMapping extends AbstractQFieldMapping<Integer>
    /*******************************************************************************
     **
     *******************************************************************************/
-   public QIndexBasedFieldMapping withMapping(Integer key, String fieldName)
+   public QIndexBasedFieldMapping withMapping(String fieldName, Integer key)
    {
-      addMapping(key, fieldName);
+      addMapping(fieldName, key);
       return (this);
    }
 
@@ -59,7 +70,7 @@ public class QIndexBasedFieldMapping extends AbstractQFieldMapping<Integer>
     ** Getter for mapping
     **
     *******************************************************************************/
-   public Map<Integer, String> getMapping()
+   public Map<String, Integer> getMapping()
    {
       return mapping;
    }
@@ -70,7 +81,7 @@ public class QIndexBasedFieldMapping extends AbstractQFieldMapping<Integer>
     ** Setter for mapping
     **
     *******************************************************************************/
-   public void setMapping(Map<Integer, String> mapping)
+   public void setMapping(Map<String, Integer> mapping)
    {
       this.mapping = mapping;
    }

src/main/java/com/kingsrook/qqq/backend/core/model/actions/QKeyBasedFieldMapping.java

@@ -18,14 +18,14 @@ public class QKeyBasedFieldMapping extends AbstractQFieldMapping<String>
     **
     *******************************************************************************/
    @Override
-   public String getMappedField(String key)
+   public String getFieldSource(String fieldName)
    {
       if(mapping == null)
       {
          return (null);
       }
 
-      return (mapping.get(key));
+      return (mapping.get(fieldName));
    }
 
 
@@ -33,13 +33,24 @@ public class QKeyBasedFieldMapping extends AbstractQFieldMapping<String>
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void addMapping(String key, String fieldName)
+   @Override
+   public SourceType getSourceType()
+   {
+      return (SourceType.KEY);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addMapping(String fieldName, String key)
    {
       if(mapping == null)
       {
          mapping = new LinkedHashMap<>();
       }
-      mapping.put(key, fieldName);
+      mapping.put(fieldName, key);
    }
 
 
@@ -47,9 +58,9 @@ public class QKeyBasedFieldMapping extends AbstractQFieldMapping<String>
    /*******************************************************************************
     **
     *******************************************************************************/
-   public QKeyBasedFieldMapping withMapping(String key, String fieldName)
+   public QKeyBasedFieldMapping withMapping(String fieldName, String key)
    {
-      addMapping(key, fieldName);
+      addMapping(fieldName, key);
       return (this);
    }
 

src/main/java/com/kingsrook/qqq/backend/core/model/actions/QueryRequest.java

@@ -1,7 +1,7 @@
 package com.kingsrook.qqq.backend.core.model.actions;
 
 
-import com.kingsrook.qqq.backend.core.model.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 
 
 /*******************************************************************************

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java

@@ -3,6 +3,7 @@ package com.kingsrook.qqq.backend.core.model.metadata;
 
 import java.util.HashMap;
 import java.util.Map;
+import com.fasterxml.jackson.annotation.JsonFilter;
 
 
 /*******************************************************************************
@@ -12,6 +13,8 @@ public class QBackendMetaData
 {
    private String name;
    private String type;
+
+   @JsonFilter("secretsFilter")
    private Map<String, String> values;
 
 

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java

@@ -1,10 +1,10 @@
-package com.kingsrook.qqq.backend.core.model;
+package com.kingsrook.qqq.backend.core.model.metadata;
 
 
 import java.util.HashMap;
 import java.util.Map;
-import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidationKey;
 
 
 /*******************************************************************************
@@ -12,9 +12,18 @@ import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
  *******************************************************************************/
 public class QInstance
 {
+   ///////////////////////////////////////////////////////////////////////////////
+   // Do not let the backend data be serialized - e.g., sent to a frontend user //
+   ///////////////////////////////////////////////////////////////////////////////
+   @JsonIgnore
    private Map<String, QBackendMetaData> backends = new HashMap<>();
+
    private Map<String, QTableMetaData> tables = new HashMap<>();
 
+   // todo - lock down the object (no more changes allowed) after it's been validated?
+   @JsonIgnore
+   private boolean hasBeenValidated = false;
+
 
 
    /*******************************************************************************
@@ -29,16 +38,27 @@ public class QInstance
       }
 
       QBackendMetaData backend = backends.get(table.getBackendName());
-      if(backend == null)
+
-      {
+      //////////////////////////////////////////////////////////////////////////////////////////////
-         throw (new IllegalArgumentException("Table [" + tableName + "] specified a backend name [" + table.getBackendName() + "] which was found in this instance."));
+      // validation should already let us know that this is valid, so no need to check/throw here //
-      }
+      //////////////////////////////////////////////////////////////////////////////////////////////
 
       return (backend);
    }
 
 
 
+   /*******************************************************************************
+    ** Setter for hasBeenValidated
+    **
+    *******************************************************************************/
+   public void setHasBeenValidated(QInstanceValidationKey key)
+   {
+      this.hasBeenValidated = true;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -140,4 +160,16 @@ public class QInstance
    {
       this.tables = tables;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for hasBeenValidated
+    **
+    *******************************************************************************/
+   public boolean getHasBeenValidated()
+   {
+      return hasBeenValidated;
+   }
+
 }

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QTableMetaData.java

@@ -1,9 +1,7 @@
 package com.kingsrook.qqq.backend.core.model.metadata;
 
 
-import java.util.ArrayList;
 import java.util.LinkedHashMap;
-import java.util.List;
 import java.util.Map;
 
 
@@ -13,10 +11,10 @@ import java.util.Map;
 public class QTableMetaData
 {
    private String name;
+   private String label;
    private String backendName;
    private String primaryKeyField;
-   private List<QFieldMetaData> fields;
+   private Map<String, QFieldMetaData> fields;
-   private Map<String, QFieldMetaData> _fieldMap;
 
 
 
@@ -30,7 +28,7 @@ public class QTableMetaData
          throw (new IllegalArgumentException("Table [" + name + "] does not have its fields defined."));
       }
 
-      QFieldMetaData field = getFieldMap().get(fieldName);
+      QFieldMetaData field = getFields().get(fieldName);
       if(field == null)
       {
          throw (new IllegalArgumentException("Field [" + fieldName + "] was not found in table [" + name + "]."));
@@ -41,25 +39,6 @@ public class QTableMetaData
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private Map<String, QFieldMetaData> getFieldMap()
-   {
-      if(_fieldMap == null)
-      {
-         _fieldMap = new LinkedHashMap<>();
-         for(QFieldMetaData field : fields)
-         {
-            _fieldMap.put(field.getName(), field);
-         }
-      }
-
-      return (_fieldMap);
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -91,6 +70,37 @@ public class QTableMetaData
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String getLabel()
+   {
+      return label;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setLabel(String label)
+   {
+      this.label = label;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QTableMetaData withLabel(String label)
+   {
+      this.label = label;
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for backendName
     **
@@ -158,7 +168,7 @@ public class QTableMetaData
    /*******************************************************************************
     **
     *******************************************************************************/
-   public List<QFieldMetaData> getFields()
+   public Map<String, QFieldMetaData> getFields()
    {
       return fields;
    }
@@ -166,9 +176,10 @@ public class QTableMetaData
 
 
    /*******************************************************************************
+    ** Setter for fields
     **
     *******************************************************************************/
-   public void setFields(List<QFieldMetaData> fields)
+   public void setFields(Map<String, QFieldMetaData> fields)
    {
       this.fields = fields;
    }
@@ -178,7 +189,7 @@ public class QTableMetaData
    /*******************************************************************************
     **
     *******************************************************************************/
-   public QTableMetaData withFields(List<QFieldMetaData> fields)
+   public QTableMetaData withFields(Map<String, QFieldMetaData> fields)
    {
       this.fields = fields;
       return (this);
@@ -193,9 +204,9 @@ public class QTableMetaData
    {
       if(this.fields == null)
       {
-         this.fields = new ArrayList<>();
+         this.fields = new LinkedHashMap<>();
       }
-      this.fields.add(field);
+      this.fields.put(field.getName(), field);
    }
 
 
@@ -207,9 +218,9 @@ public class QTableMetaData
    {
       if(this.fields == null)
       {
-         this.fields = new ArrayList<>();
+         this.fields = new LinkedHashMap<>();
       }
-      this.fields.add(field);
+      this.fields.put(field.getName(), field);
       return (this);
    }
 

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java

@@ -0,0 +1,69 @@
+package com.kingsrook.qqq.backend.core.model.metadata.frontend;
+
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
+
+
+/*******************************************************************************
+ * Version of QTableMetaData that's meant for transmitting to a frontend.
+ * e.g., it excludes backend-only details.
+ *******************************************************************************/
+@JsonInclude(Include.NON_NULL)
+public class QFrontendFieldMetaData
+{
+   private String name;
+   private String label;
+   private QFieldType type;
+
+   //////////////////////////////////////////////////////////////////////////////////
+   // do not add setters.  take values from the source-object in the constructor!! //
+   //////////////////////////////////////////////////////////////////////////////////
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QFrontendFieldMetaData(QFieldMetaData fieldMetaData)
+   {
+      this.name = fieldMetaData.getName();
+      this.label = fieldMetaData.getLabel();
+      this.type = fieldMetaData.getType();
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for name
+    **
+    *******************************************************************************/
+   public String getName()
+   {
+      return name;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for label
+    **
+    *******************************************************************************/
+   public String getLabel()
+   {
+      return label;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for type
+    **
+    *******************************************************************************/
+   public QFieldType getType()
+   {
+      return type;
+   }
+}

src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java

@@ -0,0 +1,92 @@
+package com.kingsrook.qqq.backend.core.model.metadata.frontend;
+
+
+import java.util.HashMap;
+import java.util.Map;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+
+
+/*******************************************************************************
+ * Version of QTableMetaData that's meant for transmitting to a frontend.
+ * e.g., it excludes backend-only details.
+ *******************************************************************************/
+@JsonInclude(Include.NON_NULL)
+public class QFrontendTableMetaData
+{
+   private String name;
+   private String label;
+   private String primaryKeyField;
+   private Map<String, QFrontendFieldMetaData> fields;
+
+   //////////////////////////////////////////////////////////////////////////////////
+   // do not add setters.  take values from the source-object in the constructor!! //
+   //////////////////////////////////////////////////////////////////////////////////
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QFrontendTableMetaData(QTableMetaData tableMetaData, boolean includeFields)
+   {
+      this.name = tableMetaData.getName();
+      this.label = tableMetaData.getLabel();
+
+      if(includeFields)
+      {
+         this.primaryKeyField = tableMetaData.getPrimaryKeyField();
+         this.fields = new HashMap<>();
+         for(Map.Entry<String, QFieldMetaData> entry : tableMetaData.getFields().entrySet())
+         {
+            this.fields.put(entry.getKey(), new QFrontendFieldMetaData(entry.getValue()));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for name
+    **
+    *******************************************************************************/
+   public String getName()
+   {
+      return name;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for label
+    **
+    *******************************************************************************/
+   public String getLabel()
+   {
+      return label;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for primaryKeyField
+    **
+    *******************************************************************************/
+   public String getPrimaryKeyField()
+   {
+      return primaryKeyField;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for fields
+    **
+    *******************************************************************************/
+   public Map<String, QFrontendFieldMetaData> getFields()
+   {
+      return fields;
+   }
+}

src/main/java/com/kingsrook/qqq/backend/core/modules/QModuleDispatcher.java

@@ -52,7 +52,7 @@ public class QModuleDispatcher
       }
       catch(Exception e)
       {
-         throw (new QModuleDispatchException("Error getting module", e));
+         throw (new QModuleDispatchException("Error getting q backend module of type: " + backend.getType(), e));
       }
    }
 }

src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java

@@ -30,6 +30,22 @@ public class CollectionUtils
 
 
 
+   /*******************************************************************************
+    ** true if c is null or it's empty
+    **
+    *******************************************************************************/
+   public static boolean nullSafeIsEmpty(Map c)
+   {
+      if(c == null || c.isEmpty())
+      {
+         return (true);
+      }
+
+      return (false);
+   }
+
+
+
    /*******************************************************************************
     ** true if c is NOT null and it's not empty
     **
@@ -41,6 +57,17 @@ public class CollectionUtils
 
 
 
+   /*******************************************************************************
+    ** true if c is NOT null and it's not empty
+    **
+    *******************************************************************************/
+   public static boolean nullSafeHasContents(Map c)
+   {
+      return (!nullSafeIsEmpty(c));
+   }
+
+
+
    /*******************************************************************************
     ** 0 if c is empty, otherwise, its size.
     **
@@ -57,6 +84,22 @@ public class CollectionUtils
 
 
 
+   /*******************************************************************************
+    ** 0 if c is empty, otherwise, its size.
+    **
+    *******************************************************************************/
+   public static int nullSafeSize(Map c)
+   {
+      if(c == null)
+      {
+         return (0);
+      }
+
+      return (c.size());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java

@@ -94,6 +94,15 @@ public class JsonUtils
       ObjectMapper mapper = new ObjectMapper()
          .registerModule(new JavaTimeModule())
          .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
+
+      /* todo - some future version we may need to do inclusion/exclusion lists like this:
+      // this is what we'd put on the class or member we wanted to 'filter':  @JsonFilter("secretsFilter")
+
+      SimpleFilterProvider filterProvider = new SimpleFilterProvider();
+      filterProvider.addFilter("secretsFilter", SimpleBeanPropertyFilter.serializeAllExcept("password"));
+      mapper.setFilterProvider(filterProvider);
+       */
+
       return (mapper);
    }
 

src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java

@@ -0,0 +1,222 @@
+package com.kingsrook.qqq.backend.core.adapters;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.actions.QIndexBasedFieldMapping;
+import com.kingsrook.qqq.backend.core.model.actions.QKeyBasedFieldMapping;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+class CsvToQRecordAdapterTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildRecordsFromCsv_nullInput()
+   {
+      testExpectedToThrow(null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildRecordsFromCsv_emptyStringInput()
+   {
+      testExpectedToThrow("");
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   /* todo?
+   @Test
+   public void test_buildRecordsFromCsv_inputDoesntLookLikeCsv()
+   {
+      testExpectedToThrow("CSV");
+   }
+   */
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void testExpectedToThrow(String csv)
+   {
+      try
+      {
+         CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+         List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(csv, TestUtils.defineTablePerson(), null);
+         System.out.println(qRecords);
+      }
+      catch(IllegalArgumentException iae)
+      {
+         System.out.println("Threw expected exception");
+         return;
+      }
+
+      fail("Didn't throw expected exception");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildRecordsFromCsv_emptyList()
+   {
+      CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+      List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader(), TestUtils.defineTablePerson(), null);
+      assertNotNull(qRecords);
+      assertTrue(qRecords.isEmpty());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String getPersonCsvHeader()
+   {
+      return ("""
+         "id","createDate","modifyDate","firstName","lastName","birthDate","email"\r
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String getPersonCsvRow2()
+   {
+      return ("""
+         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"\r
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String getPersonCsvRow1()
+   {
+      return ("""
+         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"\r
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildRecordsFromCsv_oneRowStandardHeaderNoMapping()
+   {
+      CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+      List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1(), TestUtils.defineTablePerson(), null);
+      assertNotNull(qRecords);
+      assertEquals(1, qRecords.size());
+      QRecord qRecord = qRecords.get(0);
+      assertEquals("John", qRecord.getValue("firstName"));
+      assertEquals("1980-01-01", qRecord.getValue("birthDate"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildRecordsFromCsv_twoRowsStandardHeaderNoMapping()
+   {
+      CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+      List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), null);
+      assertNotNull(qRecords);
+      assertEquals(2, qRecords.size());
+      QRecord qRecord1 = qRecords.get(0);
+      assertEquals("John", qRecord1.getValue("firstName"));
+      assertEquals("1980-01-01", qRecord1.getValue("birthDate"));
+      QRecord qRecord2 = qRecords.get(1);
+      assertEquals("Jane", qRecord2.getValue("firstName"));
+      assertEquals("1981-01-01", qRecord2.getValue("birthDate"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildRecordsFromCsv_oneRowCustomKeyBasedMapping()
+   {
+      String csvCustomHeader = """
+         "id","created","modified","first","last","birthday","email"\r
+         """;
+
+      QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping()
+         .withMapping("id", "id")
+         .withMapping("createDate", "created")
+         .withMapping("modifyDate", "modified")
+         .withMapping("firstName", "first")
+         .withMapping("lastName", "last")
+         .withMapping("birthDate", "birthday")
+         .withMapping("email", "email");
+
+      CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+      List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(csvCustomHeader + getPersonCsvRow1(), TestUtils.defineTablePerson(), mapping);
+      assertNotNull(qRecords);
+      assertEquals(1, qRecords.size());
+      QRecord qRecord = qRecords.get(0);
+      assertEquals("John", qRecord.getValue("firstName"));
+      assertEquals("1980-01-01", qRecord.getValue("birthDate"));
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildRecordsFromCsv_twoRowsCustomIndexBasedMapping()
+   {
+      int index = 1;
+      QIndexBasedFieldMapping mapping = new QIndexBasedFieldMapping()
+         .withMapping("id", index++)
+         .withMapping("createDate", index++)
+         .withMapping("modifyDate", index++)
+         .withMapping("firstName", index++)
+         .withMapping("lastName", index++)
+         .withMapping("birthDate", index++)
+         .withMapping("email", index++);
+
+      CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+      List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), mapping);
+      assertNotNull(qRecords);
+      assertEquals(2, qRecords.size());
+      QRecord qRecord1 = qRecords.get(0);
+      assertEquals("John", qRecord1.getValue("firstName"));
+      assertEquals("1980-01-01", qRecord1.getValue("birthDate"));
+      QRecord qRecord2 = qRecords.get(1);
+      assertEquals("Jane", qRecord2.getValue("firstName"));
+      assertEquals("1981-01-01", qRecord2.getValue("birthDate"));
+   }
+
+
+}

src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapterTest.java

@@ -2,6 +2,7 @@ package com.kingsrook.qqq.backend.core.adapters;
 
 
 import com.kingsrook.qqq.backend.core.model.actions.AbstractQFieldMapping;
+import com.kingsrook.qqq.backend.core.model.actions.QIndexBasedFieldMapping;
 import com.kingsrook.qqq.backend.core.model.actions.QKeyBasedFieldMapping;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -51,7 +52,7 @@ class JsonToQFieldMappingAdapterTest
     **
     *******************************************************************************/
    @Test
-   public void test_buildMappingFromJson_validInput()
+   public void test_buildMappingFromJson_validKeyBasedInput()
    {
       JsonToQFieldMappingAdapter jsonToQFieldMappingAdapter = new JsonToQFieldMappingAdapter();
       AbstractQFieldMapping<String> mapping = (QKeyBasedFieldMapping) jsonToQFieldMappingAdapter.buildMappingFromJson("""
@@ -63,9 +64,67 @@ class JsonToQFieldMappingAdapterTest
       System.out.println(mapping);
       assertNotNull(mapping);
 
-      // todo - are we backwards here??
+      assertEquals("source1", mapping.getFieldSource("Field1"));
-      assertEquals("source1", mapping.getMappedField("Field1"));
+      assertEquals("source2", mapping.getFieldSource("Field2"));
-      assertEquals("source2", mapping.getMappedField("Field2"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildMappingFromJson_validIndexBasedInput()
+   {
+      JsonToQFieldMappingAdapter jsonToQFieldMappingAdapter = new JsonToQFieldMappingAdapter();
+      AbstractQFieldMapping<Integer> mapping = (QIndexBasedFieldMapping) jsonToQFieldMappingAdapter.buildMappingFromJson("""
+         {
+            "Field1": 1,
+            "Field2": 2,
+         }
+         """);
+      System.out.println(mapping);
+      assertNotNull(mapping);
+
+      assertEquals(1, mapping.getFieldSource("Field1"));
+      assertEquals(2, mapping.getFieldSource("Field2"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildMappingFromJson_unsupportedTypeForSource()
+   {
+      testExpectedToThrow("""
+         {
+            "Field1": [1, 2],
+            "Field2": {"A": "B"}
+         }
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildMappingFromJson_emptyMapping()
+   {
+      testExpectedToThrow("{}");
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildMappingFromJson_inputJsonList()
+   {
+      testExpectedToThrow("[]");
    }
 
 

src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQRecordAdapterTest.java

@@ -3,6 +3,7 @@ package com.kingsrook.qqq.backend.core.adapters;
 
 import java.util.List;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.*;
 
@@ -65,7 +66,7 @@ class JsonToQRecordAdapterTest
       try
       {
          JsonToQRecordAdapter jsonToQRecordAdapter = new JsonToQRecordAdapter();
-         List<QRecord> qRecords = jsonToQRecordAdapter.buildRecordsFromJson(json);
+         List<QRecord> qRecords = jsonToQRecordAdapter.buildRecordsFromJson(json, TestUtils.defineTablePerson(), null);
          System.out.println(qRecords);
       }
       catch(IllegalArgumentException iae)
@@ -86,7 +87,7 @@ class JsonToQRecordAdapterTest
    public void test_buildRecordsFromJson_emptyList()
    {
       JsonToQRecordAdapter jsonToQRecordAdapter = new JsonToQRecordAdapter();
-      List<QRecord> qRecords = jsonToQRecordAdapter.buildRecordsFromJson("[]");
+      List<QRecord> qRecords = jsonToQRecordAdapter.buildRecordsFromJson("[]", TestUtils.defineTablePerson(), null);
       assertNotNull(qRecords);
       assertTrue(qRecords.isEmpty());
    }
@@ -105,7 +106,7 @@ class JsonToQRecordAdapterTest
             "field1":"value1",
             "field2":"value2"
          }
-         """);
+         """, TestUtils.defineTablePerson(), null);
       assertNotNull(qRecords);
       assertEquals(1, qRecords.size());
       assertEquals("value1", qRecords.get(0).getValue("field1"));
@@ -126,7 +127,7 @@ class JsonToQRecordAdapterTest
             { "field1":"value1", "field2":"value2" },
             { "fieldA":"valueA", "fieldB":"valueB" }
          ]
-         """);
+         """, TestUtils.defineTablePerson(), null);
       assertNotNull(qRecords);
       assertEquals(2, qRecords.size());
       assertEquals("value1", qRecords.get(0).getValue("field1"));

src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -0,0 +1,188 @@
+package com.kingsrook.qqq.backend.core.instances;
+
+
+import java.util.HashMap;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+class QInstanceValidatorTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_validatePass() throws QInstanceValidationException
+   {
+      new QInstanceValidator().validate(TestUtils.defineInstance());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_validateNullBackends()
+   {
+      try
+      {
+         QInstance qInstance = TestUtils.defineInstance();
+         qInstance.setBackends(null);
+         new QInstanceValidator().validate(qInstance);
+         fail("Should have thrown validationException");
+      }
+      catch(QInstanceValidationException e)
+      {
+         assertReason("At least 1 backend must be defined", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_validateEmptyBackends()
+   {
+      try
+      {
+         QInstance qInstance = TestUtils.defineInstance();
+         qInstance.setBackends(new HashMap<>());
+         new QInstanceValidator().validate(qInstance);
+         fail("Should have thrown validationException");
+      }
+      catch(QInstanceValidationException e)
+      {
+         assertReason("At least 1 backend must be defined", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_validateNullTables()
+   {
+      try
+      {
+         QInstance qInstance = TestUtils.defineInstance();
+         qInstance.setTables(null);
+         new QInstanceValidator().validate(qInstance);
+         fail("Should have thrown validationException");
+      }
+      catch(QInstanceValidationException e)
+      {
+         assertReason("At least 1 table must be defined", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_validateEmptyTables()
+   {
+      try
+      {
+         QInstance qInstance = TestUtils.defineInstance();
+         qInstance.setTables(new HashMap<>());
+         new QInstanceValidator().validate(qInstance);
+         fail("Should have thrown validationException");
+      }
+      catch(QInstanceValidationException e)
+      {
+         assertReason("At least 1 table must be defined", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_validateInconsistentNames()
+   {
+      try
+      {
+         QInstance qInstance = TestUtils.defineInstance();
+         qInstance.getTable("person").setName("notPerson");
+         qInstance.getBackend("default").setName("notDefault");
+         new QInstanceValidator().validate(qInstance);
+         fail("Should have thrown validationException");
+      }
+      catch(QInstanceValidationException e)
+      {
+         assertReason("Inconsistent naming for table", e);
+         assertReason("Inconsistent naming for backend", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_validateTableWithoutBackend()
+   {
+      try
+      {
+         QInstance qInstance = TestUtils.defineInstance();
+         qInstance.getTable("person").setBackendName(null);
+         new QInstanceValidator().validate(qInstance);
+         fail("Should have thrown validationException");
+      }
+      catch(QInstanceValidationException e)
+      {
+         assertReason("Missing backend name for table", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_validateTableWithMissingBackend()
+   {
+      try
+      {
+         QInstance qInstance = TestUtils.defineInstance();
+         qInstance.getTable("person").setBackendName("notARealBackend");
+         new QInstanceValidator().validate(qInstance);
+         fail("Should have thrown validationException");
+      }
+      catch(QInstanceValidationException e)
+      {
+         assertReason("Unrecognized backend", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void assertReason(String reason, QInstanceValidationException e)
+   {
+      assertNotNull(e.getReasons());
+      assertTrue(e.getReasons().stream().anyMatch(s -> s.contains(reason)));
+   }
+}

src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -0,0 +1,65 @@
+package com.kingsrook.qqq.backend.core.utils;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class TestUtils
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QInstance defineInstance()
+   {
+      QInstance qInstance = new QInstance();
+      qInstance.addBackend(defineBackend());
+      qInstance.addTable(defineTablePerson());
+      return (qInstance);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QBackendMetaData defineBackend()
+   {
+      return new QBackendMetaData()
+         .withName("default")
+         .withType("rdbms")
+         .withValue("vendor", "h2")
+         .withValue("hostName", "mem")
+         .withValue("databaseName", "test_database")
+         .withValue("username", "sa")
+         .withValue("password", "");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QTableMetaData defineTablePerson()
+   {
+      return new QTableMetaData()
+         .withName("person")
+         .withLabel("Person")
+         .withBackendName(defineBackend().getName())
+         .withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))
+         .withField(new QFieldMetaData("firstName", QFieldType.STRING))
+         .withField(new QFieldMetaData("lastName", QFieldType.STRING))
+         .withField(new QFieldMetaData("birthDate", QFieldType.DATE))
+         .withField(new QFieldMetaData("email", QFieldType.STRING));
+   }
+
+}