Checkpoint on loaders

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/AbstractMetaDataLoader.java

@@ -31,20 +31,23 @@ import java.lang.reflect.Type;
 import java.math.BigDecimal;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QMetaDataObject;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.core.utils.YamlUtils;
 import org.apache.commons.io.IOUtils;
-import static com.kingsrook.qqq.backend.core.utils.ValueUtils.getValueAsBoolean;
 import static com.kingsrook.qqq.backend.core.utils.ValueUtils.getValueAsInteger;
 import static com.kingsrook.qqq.backend.core.utils.ValueUtils.getValueAsString;
 
@@ -59,6 +62,8 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
 
    private String fileName;
 
+   private List<LoadingProblem> problems = new ArrayList<>();
+
 
 
    /***************************************************************************
@@ -68,7 +73,8 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
    {
       this.fileName = fileName;
       Map<String, Object> map = fileToMap(inputStream, fileName);
-      return (mapToMetaDataObject(qInstance, map));
+      LoadingContext loadingContext = new LoadingContext(fileName, "/");
+      return (mapToMetaDataObject(qInstance, map, loadingContext));
    }
 
 
@@ -76,7 +82,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
    /***************************************************************************
     **
     ***************************************************************************/
-   public abstract T mapToMetaDataObject(QInstance qInstance, Map<String, Object> map) throws QMetaDataLoaderException;
+   public abstract T mapToMetaDataObject(QInstance qInstance, Map<String, Object> map, LoadingContext context) throws QMetaDataLoaderException;
 
 
 
@@ -111,9 +117,10 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
    /***************************************************************************
     *
     ***************************************************************************/
-   protected void reflectivelyMap(QInstance qInstance, QMetaDataObject targetObject, Map<String, Object> map)
+   protected void reflectivelyMap(QInstance qInstance, QMetaDataObject targetObject, Map<String, Object> map, LoadingContext context)
    {
       Class<? extends QMetaDataObject> targetClass    = targetObject.getClass();
+      Set<String>                      usedFieldNames = new HashSet<>();
 
       for(Method method : targetClass.getMethods())
       {
@@ -125,12 +132,13 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
 
                if(map.containsKey(propertyName))
                {
+                  usedFieldNames.add(propertyName);
                   Class<?> parameterType = method.getParameterTypes()[0];
                   Object   rawValue      = map.get(propertyName);
 
                   try
                   {
-                     Object mappedValue = reflectivelyMapValue(qInstance, method, parameterType, rawValue);
+                     Object mappedValue = reflectivelyMapValue(qInstance, method, parameterType, rawValue, context.descendToProperty(propertyName));
                      method.invoke(targetObject, mappedValue);
                   }
                   catch(NoValueException nve)
@@ -138,15 +146,30 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
                      ///////////////////////
                      // don't call setter //
                      ///////////////////////
+                     LOG.debug("at " + context + ": No value was mapped for property [" + propertyName + "] on " + targetClass.getSimpleName() + "." + method.getName() + ", raw value: [" + rawValue + "]");
                   }
                }
             }
          }
          catch(Exception e)
          {
-            LOG.warn("Error reflectively mapping on " + targetClass.getName() + "." + method.getName(), e);
+            addProblem(new LoadingProblem(context, "Error reflectively mapping on " + targetClass.getName() + "." + method.getName(), e));
          }
       }
+
+      //////////////////////////
+      // mmm, slightly sus... //
+      //////////////////////////
+      map.remove("class");
+      map.remove("version");
+
+      Set<String> unrecognizedKeys = new HashSet<>(map.keySet());
+      unrecognizedKeys.removeAll(usedFieldNames);
+
+      if(!unrecognizedKeys.isEmpty())
+      {
+         addProblem(new LoadingProblem(context, unrecognizedKeys.size() + " Unrecognized " + StringUtils.plural(unrecognizedKeys, "property", "properties") + ": " + unrecognizedKeys));
+      }
    }
 
 
@@ -154,26 +177,63 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
    /***************************************************************************
     *
     ***************************************************************************/
-   public Object reflectivelyMapValue(QInstance qInstance, Method method, Class<?> parameterType, Object rawValue) throws Exception
+   public Object reflectivelyMapValue(QInstance qInstance, Method method, Class<?> parameterType, Object rawValue, LoadingContext context) throws Exception
    {
+      if(rawValue instanceof String s && s.matches("^\\$\\{.+\\..+}"))
+      {
+         rawValue = new QMetaDataVariableInterpreter().interpret(s);
+         LOG.debug("Interpreted raw value [" + s + "] as [" + StringUtils.maskAndTruncate(ValueUtils.getValueAsString(rawValue) + "]"));
+      }
+
       if(parameterType.equals(String.class))
       {
          return (getValueAsString(rawValue));
       }
       else if(parameterType.equals(Integer.class))
+      {
+         try
          {
             return (getValueAsInteger(rawValue));
          }
+         catch(Exception e)
+         {
+            addProblem(new LoadingProblem(context, "[" + rawValue + "] is not an Integer value."));
+         }
+      }
       else if(parameterType.equals(Boolean.class))
       {
-         return (getValueAsBoolean(rawValue));
+         if("true".equals(rawValue) || Boolean.TRUE.equals(rawValue))
+         {
+            return (true);
+         }
+         else if("false".equals(rawValue) || Boolean.FALSE.equals(rawValue))
+         {
+            return (false);
+         }
+         else if(rawValue == null)
+         {
+            return (null);
+         }
+         else
+         {
+            addProblem(new LoadingProblem(context, "[" + rawValue + "] is not a boolean value (must be 'true' or 'false')."));
+            return (null);
+         }
       }
       else if(parameterType.equals(boolean.class))
       {
-         Boolean valueAsBoolean = getValueAsBoolean(rawValue);
+         if("true".equals(rawValue) || Boolean.TRUE.equals(rawValue))
-         if(valueAsBoolean != null)
          {
-            return (valueAsBoolean);
+            return (true);
+         }
+         else if("false".equals(rawValue) || Boolean.FALSE.equals(rawValue))
+         {
+            return (false);
+         }
+         else
+         {
+            addProblem(new LoadingProblem(context, rawValue + " is not a boolean value (must be 'true' or 'false')."));
+            throw (new NoValueException());
          }
       }
       else if(parameterType.equals(List.class))
@@ -188,7 +248,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
             {
                try
                {
-                  Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, o);
+                  Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, o, context);
                   mappedValueList.add(mappedValue);
                }
                catch(NoValueException nve)
@@ -211,7 +271,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
             {
                try
                {
-                  Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, o);
+                  Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, o, context);
                   mappedValueSet.add(mappedValue);
                }
                catch(NoValueException nve)
@@ -227,7 +287,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
          Type keyType = ((ParameterizedType) method.getGenericParameterTypes()[0]).getActualTypeArguments()[0];
          if(!keyType.equals(String.class))
          {
-            LOG.warn("Unsupported key type for " + method + " (" + keyType + ")");
+            addProblem(new LoadingProblem(context, "Unsupported key type for " + method + " got [" + keyType + "], expected [String]"));
             throw new NoValueException();
          }
          // todo make sure string
@@ -244,7 +304,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
                {
                   @SuppressWarnings("unchecked")
                   Map.Entry<String, Object> entry = (Map.Entry<String, Object>) o;
-                  Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, entry.getValue());
+                  Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, entry.getValue(), context);
                   mappedValueMap.put(entry.getKey(), mappedValue);
                }
                catch(NoValueException nve)
@@ -265,6 +325,8 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
                return (enumConstant);
             }
          }
+
+         addProblem(new LoadingProblem(context, "Unrecognized value [" + rawValue + "].  Expected one of: " + Arrays.toString(parameterType.getEnumConstants())));
       }
       else if(MetaDataLoaderRegistry.hasLoaderForClass(parameterType))
       {
@@ -273,7 +335,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
             Class<? extends AbstractMetaDataLoader<?>> loaderClass = MetaDataLoaderRegistry.getLoaderForClass(parameterType);
             AbstractMetaDataLoader<?>                  loader      = loaderClass.getConstructor().newInstance();
             //noinspection unchecked
-            return (loader.mapToMetaDataObject(qInstance, valueMap));
+            return (loader.mapToMetaDataObject(qInstance, valueMap, context));
          }
       }
       else if(QMetaDataObject.class.isAssignableFrom(parameterType))
@@ -282,7 +344,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
          {
             QMetaDataObject childObject = (QMetaDataObject) parameterType.getConstructor().newInstance();
             //noinspection unchecked
-            reflectivelyMap(qInstance, childObject, valueMap);
+            reflectivelyMap(qInstance, childObject, valueMap, context);
             return (childObject);
          }
       }
@@ -300,7 +362,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
       else
       {
          // todo clean up this message/level
-         LOG.warn("No case for " + parameterType + " (arg to: " + method + ")");
+         addProblem(new LoadingProblem(context, "No case for " + parameterType + " (arg to: " + method + ")"));
       }
 
       throw new NoValueException();
@@ -424,4 +486,25 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
          super("No value");
       }
    }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public void addProblem(LoadingProblem problem)
+   {
+      problems.add(problem);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for problems
+    **
+    *******************************************************************************/
+   public List<LoadingProblem> getProblems()
+   {
+      return (problems);
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/ClassDetectingMetaDataLoader.java

@@ -25,11 +25,14 @@ package com.kingsrook.qqq.backend.core.instances.loaders;
 import java.io.InputStream;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.instances.loaders.implementations.GenericMetaDataLoader;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QMetaDataObject;
 import com.kingsrook.qqq.backend.core.utils.ClassPathUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import com.kingsrook.qqq.backend.core.utils.memoization.AnyKey;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 
 
 /*******************************************************************************
@@ -39,6 +42,8 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
  *******************************************************************************/
 public class ClassDetectingMetaDataLoader extends AbstractMetaDataLoader<QMetaDataObject>
 {
+   private static final Memoization<AnyKey, List<Class<?>>> memoizedMetaDataObjectClasses = new Memoization<>();
+
 
    /***************************************************************************
     *
@@ -61,7 +66,6 @@ public class ClassDetectingMetaDataLoader extends AbstractMetaDataLoader<QMetaDa
          String classProperty = ValueUtils.getValueAsString(map.get("class"));
          try
          {
-
             if(MetaDataLoaderRegistry.hasLoaderForSimpleName(classProperty))
             {
                Class<? extends AbstractMetaDataLoader<?>> loaderClass = MetaDataLoaderRegistry.getLoaderForSimpleName(classProperty);
@@ -69,8 +73,13 @@ public class ClassDetectingMetaDataLoader extends AbstractMetaDataLoader<QMetaDa
             }
             else
             {
-               List<Class<?>> classesInPackage = ClassPathUtils.getClassesInPackage("com.kingsrook.qqq.backend.core.model");
+               Optional<List<Class<?>>> metaDataClasses = memoizedMetaDataObjectClasses.getResult(AnyKey.getInstance(), k -> ClassPathUtils.getClassesContainingNameAndOfType("MetaData", QMetaDataObject.class));
-               for(Class<?> c : classesInPackage)
+               if(metaDataClasses.isEmpty())
+               {
+                  throw (new QMetaDataLoaderException("Could not get list of metaDataObjects from class loader"));
+               }
+
+               for(Class<?> c : metaDataClasses.get())
                {
                   if(c.getSimpleName().equals(classProperty) && QMetaDataObject.class.isAssignableFrom(c))
                   {
@@ -103,9 +112,9 @@ public class ClassDetectingMetaDataLoader extends AbstractMetaDataLoader<QMetaDa
     **
     ***************************************************************************/
    @Override
-   public QMetaDataObject mapToMetaDataObject(QInstance qInstance, Map<String, Object> map) throws QMetaDataLoaderException
+   public QMetaDataObject mapToMetaDataObject(QInstance qInstance, Map<String, Object> map, LoadingContext context) throws QMetaDataLoaderException
    {
       AbstractMetaDataLoader<?> loaderForMap = getLoaderForMap(map);
-      return loaderForMap.mapToMetaDataObject(qInstance, map);
+      return loaderForMap.mapToMetaDataObject(qInstance, map, context);
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/LoadingContext.java

@@ -0,0 +1,38 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.loaders;
+
+
+/*******************************************************************************
+ ** Record to track where loader objects are - e.g., what file they're on,
+ ** and at what property path within the file (e.g., helps report problems).
+ *******************************************************************************/
+public record LoadingContext(String fileName, String propertyPath)
+{
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public LoadingContext descendToProperty(String propertyName)
+   {
+      return new LoadingContext(fileName, propertyPath + propertyName + "/");
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/LoadingProblem.java

@@ -0,0 +1,49 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.loaders;
+
+
+/*******************************************************************************
+ ** record that tracks a problem that was encountered when loading files.
+ *******************************************************************************/
+public record LoadingProblem(LoadingContext context, String message, Exception exception) // todo Level if useful
+{
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public LoadingProblem(LoadingContext context, String message)
+   {
+      this(context, message, null);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public String toString()
+   {
+      return "at[" + context.fileName() + "][" + context.propertyPath() + "]: " + message;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/MetaDataLoaderHelper.java

@@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QMetaDataObject;
 import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
 
 
@@ -64,6 +65,12 @@ public class MetaDataLoaderHelper
          try(FileInputStream fileInputStream = new FileInputStream(file))
          {
             QMetaDataObject qMetaDataObject = loader.fileToMetaDataObject(qInstance, fileInputStream, file.getName());
+
+            if(CollectionUtils.nullSafeHasContents(loader.getProblems()))
+            {
+               loader.getProblems().forEach(System.out::println);
+            }
+
             if(qMetaDataObject instanceof TopLevelMetaDataInterface topLevelMetaData)
             {
                topLevelMetaData.addSelfToInstance(qInstance);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/implementations/GenericMetaDataLoader.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.instances.loaders.implementations;
 
 
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.instances.loaders.LoadingContext;
 import com.kingsrook.qqq.backend.core.instances.loaders.AbstractMetaDataLoader;
 import com.kingsrook.qqq.backend.core.instances.loaders.QMetaDataLoaderException;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -53,12 +54,12 @@ public class GenericMetaDataLoader<T extends QMetaDataObject> extends AbstractMe
     **
     ***************************************************************************/
    @Override
-   public T mapToMetaDataObject(QInstance qInstance, Map<String, Object> map) throws QMetaDataLoaderException
+   public T mapToMetaDataObject(QInstance qInstance, Map<String, Object> map, LoadingContext context) throws QMetaDataLoaderException
    {
       try
       {
          T object = metaDataClass.getConstructor().newInstance();
-         reflectivelyMap(qInstance, object, map);
+         reflectivelyMap(qInstance, object, map, context);
          return (object);
       }
       catch(Exception e)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/implementations/QTableMetaDataLoader.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.instances.loaders.implementations;
 
 
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.instances.loaders.LoadingContext;
 import com.kingsrook.qqq.backend.core.instances.loaders.AbstractMetaDataLoader;
 import com.kingsrook.qqq.backend.core.instances.loaders.QMetaDataLoaderException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -43,11 +44,11 @@ public class QTableMetaDataLoader extends AbstractMetaDataLoader<QTableMetaData>
     **
     ***************************************************************************/
    @Override
-   public QTableMetaData mapToMetaDataObject(QInstance qInstance, Map<String, Object> map) throws QMetaDataLoaderException
+   public QTableMetaData mapToMetaDataObject(QInstance qInstance, Map<String, Object> map, LoadingContext context) throws QMetaDataLoaderException
    {
       QTableMetaData table = new QTableMetaData();
 
-      reflectivelyMap(qInstance, table, map);
+      reflectivelyMap(qInstance, table, map, context);
 
       // todo - handle QTableBackendDetails, based on backend's type