From 93c7fbca25d6162ce9869468385d047b255985a2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 23 Jan 2025 09:39:31 -0600 Subject: [PATCH] Checkpoint on loaders --- .../loaders/AbstractMetaDataLoader.java | 123 +++++++++++++++--- .../loaders/ClassDetectingMetaDataLoader.java | 19 ++- .../instances/loaders/LoadingContext.java | 38 ++++++ .../instances/loaders/LoadingProblem.java | 49 +++++++ .../loaders/MetaDataLoaderHelper.java | 7 + .../GenericMetaDataLoader.java | 5 +- .../implementations/QTableMetaDataLoader.java | 5 +- 7 files changed, 217 insertions(+), 29 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/LoadingContext.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/LoadingProblem.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/AbstractMetaDataLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/AbstractMetaDataLoader.java index dc4df863..ffd659a8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/AbstractMetaDataLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/AbstractMetaDataLoader.java @@ -31,20 +31,23 @@ import java.lang.reflect.Type; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; import com.fasterxml.jackson.core.type.TypeReference; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QMetaDataObject; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.YamlUtils; import org.apache.commons.io.IOUtils; -import static com.kingsrook.qqq.backend.core.utils.ValueUtils.getValueAsBoolean; import static com.kingsrook.qqq.backend.core.utils.ValueUtils.getValueAsInteger; import static com.kingsrook.qqq.backend.core.utils.ValueUtils.getValueAsString; @@ -59,6 +62,8 @@ public abstract class AbstractMetaDataLoader private String fileName; + private List problems = new ArrayList<>(); + /*************************************************************************** @@ -68,7 +73,8 @@ public abstract class AbstractMetaDataLoader { this.fileName = fileName; Map map = fileToMap(inputStream, fileName); - return (mapToMetaDataObject(qInstance, map)); + LoadingContext loadingContext = new LoadingContext(fileName, "/"); + return (mapToMetaDataObject(qInstance, map, loadingContext)); } @@ -76,7 +82,7 @@ public abstract class AbstractMetaDataLoader /*************************************************************************** ** ***************************************************************************/ - public abstract T mapToMetaDataObject(QInstance qInstance, Map map) throws QMetaDataLoaderException; + public abstract T mapToMetaDataObject(QInstance qInstance, Map map, LoadingContext context) throws QMetaDataLoaderException; @@ -111,9 +117,10 @@ public abstract class AbstractMetaDataLoader /*************************************************************************** * ***************************************************************************/ - protected void reflectivelyMap(QInstance qInstance, QMetaDataObject targetObject, Map map) + protected void reflectivelyMap(QInstance qInstance, QMetaDataObject targetObject, Map map, LoadingContext context) { - Class targetClass = targetObject.getClass(); + Class targetClass = targetObject.getClass(); + Set usedFieldNames = new HashSet<>(); for(Method method : targetClass.getMethods()) { @@ -125,12 +132,13 @@ public abstract class AbstractMetaDataLoader if(map.containsKey(propertyName)) { + usedFieldNames.add(propertyName); Class parameterType = method.getParameterTypes()[0]; Object rawValue = map.get(propertyName); try { - Object mappedValue = reflectivelyMapValue(qInstance, method, parameterType, rawValue); + Object mappedValue = reflectivelyMapValue(qInstance, method, parameterType, rawValue, context.descendToProperty(propertyName)); method.invoke(targetObject, mappedValue); } catch(NoValueException nve) @@ -138,15 +146,30 @@ public abstract class AbstractMetaDataLoader /////////////////////// // don't call setter // /////////////////////// + LOG.debug("at " + context + ": No value was mapped for property [" + propertyName + "] on " + targetClass.getSimpleName() + "." + method.getName() + ", raw value: [" + rawValue + "]"); } } } } catch(Exception e) { - LOG.warn("Error reflectively mapping on " + targetClass.getName() + "." + method.getName(), e); + addProblem(new LoadingProblem(context, "Error reflectively mapping on " + targetClass.getName() + "." + method.getName(), e)); } } + + ////////////////////////// + // mmm, slightly sus... // + ////////////////////////// + map.remove("class"); + map.remove("version"); + + Set unrecognizedKeys = new HashSet<>(map.keySet()); + unrecognizedKeys.removeAll(usedFieldNames); + + if(!unrecognizedKeys.isEmpty()) + { + addProblem(new LoadingProblem(context, unrecognizedKeys.size() + " Unrecognized " + StringUtils.plural(unrecognizedKeys, "property", "properties") + ": " + unrecognizedKeys)); + } } @@ -154,26 +177,63 @@ public abstract class AbstractMetaDataLoader /*************************************************************************** * ***************************************************************************/ - public Object reflectivelyMapValue(QInstance qInstance, Method method, Class parameterType, Object rawValue) throws Exception + public Object reflectivelyMapValue(QInstance qInstance, Method method, Class parameterType, Object rawValue, LoadingContext context) throws Exception { + if(rawValue instanceof String s && s.matches("^\\$\\{.+\\..+}")) + { + rawValue = new QMetaDataVariableInterpreter().interpret(s); + LOG.debug("Interpreted raw value [" + s + "] as [" + StringUtils.maskAndTruncate(ValueUtils.getValueAsString(rawValue) + "]")); + } + if(parameterType.equals(String.class)) { return (getValueAsString(rawValue)); } else if(parameterType.equals(Integer.class)) { - return (getValueAsInteger(rawValue)); + try + { + return (getValueAsInteger(rawValue)); + } + catch(Exception e) + { + addProblem(new LoadingProblem(context, "[" + rawValue + "] is not an Integer value.")); + } } else if(parameterType.equals(Boolean.class)) { - return (getValueAsBoolean(rawValue)); + if("true".equals(rawValue) || Boolean.TRUE.equals(rawValue)) + { + return (true); + } + else if("false".equals(rawValue) || Boolean.FALSE.equals(rawValue)) + { + return (false); + } + else if(rawValue == null) + { + return (null); + } + else + { + addProblem(new LoadingProblem(context, "[" + rawValue + "] is not a boolean value (must be 'true' or 'false').")); + return (null); + } } else if(parameterType.equals(boolean.class)) { - Boolean valueAsBoolean = getValueAsBoolean(rawValue); - if(valueAsBoolean != null) + if("true".equals(rawValue) || Boolean.TRUE.equals(rawValue)) { - return (valueAsBoolean); + return (true); + } + else if("false".equals(rawValue) || Boolean.FALSE.equals(rawValue)) + { + return (false); + } + else + { + addProblem(new LoadingProblem(context, rawValue + " is not a boolean value (must be 'true' or 'false').")); + throw (new NoValueException()); } } else if(parameterType.equals(List.class)) @@ -188,7 +248,7 @@ public abstract class AbstractMetaDataLoader { try { - Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, o); + Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, o, context); mappedValueList.add(mappedValue); } catch(NoValueException nve) @@ -211,7 +271,7 @@ public abstract class AbstractMetaDataLoader { try { - Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, o); + Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, o, context); mappedValueSet.add(mappedValue); } catch(NoValueException nve) @@ -227,7 +287,7 @@ public abstract class AbstractMetaDataLoader Type keyType = ((ParameterizedType) method.getGenericParameterTypes()[0]).getActualTypeArguments()[0]; if(!keyType.equals(String.class)) { - LOG.warn("Unsupported key type for " + method + " (" + keyType + ")"); + addProblem(new LoadingProblem(context, "Unsupported key type for " + method + " got [" + keyType + "], expected [String]")); throw new NoValueException(); } // todo make sure string @@ -244,7 +304,7 @@ public abstract class AbstractMetaDataLoader { @SuppressWarnings("unchecked") Map.Entry entry = (Map.Entry) o; - Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, entry.getValue()); + Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, entry.getValue(), context); mappedValueMap.put(entry.getKey(), mappedValue); } catch(NoValueException nve) @@ -265,6 +325,8 @@ public abstract class AbstractMetaDataLoader return (enumConstant); } } + + addProblem(new LoadingProblem(context, "Unrecognized value [" + rawValue + "]. Expected one of: " + Arrays.toString(parameterType.getEnumConstants()))); } else if(MetaDataLoaderRegistry.hasLoaderForClass(parameterType)) { @@ -273,7 +335,7 @@ public abstract class AbstractMetaDataLoader Class> loaderClass = MetaDataLoaderRegistry.getLoaderForClass(parameterType); AbstractMetaDataLoader loader = loaderClass.getConstructor().newInstance(); //noinspection unchecked - return (loader.mapToMetaDataObject(qInstance, valueMap)); + return (loader.mapToMetaDataObject(qInstance, valueMap, context)); } } else if(QMetaDataObject.class.isAssignableFrom(parameterType)) @@ -282,7 +344,7 @@ public abstract class AbstractMetaDataLoader { QMetaDataObject childObject = (QMetaDataObject) parameterType.getConstructor().newInstance(); //noinspection unchecked - reflectivelyMap(qInstance, childObject, valueMap); + reflectivelyMap(qInstance, childObject, valueMap, context); return (childObject); } } @@ -300,7 +362,7 @@ public abstract class AbstractMetaDataLoader else { // todo clean up this message/level - LOG.warn("No case for " + parameterType + " (arg to: " + method + ")"); + addProblem(new LoadingProblem(context, "No case for " + parameterType + " (arg to: " + method + ")")); } throw new NoValueException(); @@ -424,4 +486,25 @@ public abstract class AbstractMetaDataLoader super("No value"); } } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public void addProblem(LoadingProblem problem) + { + problems.add(problem); + } + + + + /******************************************************************************* + ** Getter for problems + ** + *******************************************************************************/ + public List getProblems() + { + return (problems); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/ClassDetectingMetaDataLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/ClassDetectingMetaDataLoader.java index f5621bc3..ab89d608 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/ClassDetectingMetaDataLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/ClassDetectingMetaDataLoader.java @@ -25,11 +25,14 @@ package com.kingsrook.qqq.backend.core.instances.loaders; import java.io.InputStream; import java.util.List; import java.util.Map; +import java.util.Optional; import com.kingsrook.qqq.backend.core.instances.loaders.implementations.GenericMetaDataLoader; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QMetaDataObject; import com.kingsrook.qqq.backend.core.utils.ClassPathUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.AnyKey; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; /******************************************************************************* @@ -39,6 +42,8 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class ClassDetectingMetaDataLoader extends AbstractMetaDataLoader { + private static final Memoization>> memoizedMetaDataObjectClasses = new Memoization<>(); + /*************************************************************************** * @@ -61,7 +66,6 @@ public class ClassDetectingMetaDataLoader extends AbstractMetaDataLoader> loaderClass = MetaDataLoaderRegistry.getLoaderForSimpleName(classProperty); @@ -69,8 +73,13 @@ public class ClassDetectingMetaDataLoader extends AbstractMetaDataLoader> classesInPackage = ClassPathUtils.getClassesInPackage("com.kingsrook.qqq.backend.core.model"); - for(Class c : classesInPackage) + Optional>> metaDataClasses = memoizedMetaDataObjectClasses.getResult(AnyKey.getInstance(), k -> ClassPathUtils.getClassesContainingNameAndOfType("MetaData", QMetaDataObject.class)); + if(metaDataClasses.isEmpty()) + { + throw (new QMetaDataLoaderException("Could not get list of metaDataObjects from class loader")); + } + + for(Class c : metaDataClasses.get()) { if(c.getSimpleName().equals(classProperty) && QMetaDataObject.class.isAssignableFrom(c)) { @@ -103,9 +112,9 @@ public class ClassDetectingMetaDataLoader extends AbstractMetaDataLoader map) throws QMetaDataLoaderException + public QMetaDataObject mapToMetaDataObject(QInstance qInstance, Map map, LoadingContext context) throws QMetaDataLoaderException { AbstractMetaDataLoader loaderForMap = getLoaderForMap(map); - return loaderForMap.mapToMetaDataObject(qInstance, map); + return loaderForMap.mapToMetaDataObject(qInstance, map, context); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/LoadingContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/LoadingContext.java new file mode 100644 index 00000000..3b3b3eb1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/LoadingContext.java @@ -0,0 +1,38 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.instances.loaders; + + +/******************************************************************************* + ** Record to track where loader objects are - e.g., what file they're on, + ** and at what property path within the file (e.g., helps report problems). + *******************************************************************************/ +public record LoadingContext(String fileName, String propertyPath) +{ + /*************************************************************************** + ** + ***************************************************************************/ + public LoadingContext descendToProperty(String propertyName) + { + return new LoadingContext(fileName, propertyPath + propertyName + "/"); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/LoadingProblem.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/LoadingProblem.java new file mode 100644 index 00000000..ac783697 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/LoadingProblem.java @@ -0,0 +1,49 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.instances.loaders; + + +/******************************************************************************* + ** record that tracks a problem that was encountered when loading files. + *******************************************************************************/ +public record LoadingProblem(LoadingContext context, String message, Exception exception) // todo Level if useful +{ + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public LoadingProblem(LoadingContext context, String message) + { + this(context, message, null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String toString() + { + return "at[" + context.fileName() + "][" + context.propertyPath() + "]: " + message; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/MetaDataLoaderHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/MetaDataLoaderHelper.java index a872c04d..54a78064 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/MetaDataLoaderHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/MetaDataLoaderHelper.java @@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QMetaDataObject; import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; @@ -64,6 +65,12 @@ public class MetaDataLoaderHelper try(FileInputStream fileInputStream = new FileInputStream(file)) { QMetaDataObject qMetaDataObject = loader.fileToMetaDataObject(qInstance, fileInputStream, file.getName()); + + if(CollectionUtils.nullSafeHasContents(loader.getProblems())) + { + loader.getProblems().forEach(System.out::println); + } + if(qMetaDataObject instanceof TopLevelMetaDataInterface topLevelMetaData) { topLevelMetaData.addSelfToInstance(qInstance); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/implementations/GenericMetaDataLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/implementations/GenericMetaDataLoader.java index 20440c9a..42c19852 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/implementations/GenericMetaDataLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/implementations/GenericMetaDataLoader.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.instances.loaders.implementations; import java.util.Map; +import com.kingsrook.qqq.backend.core.instances.loaders.LoadingContext; import com.kingsrook.qqq.backend.core.instances.loaders.AbstractMetaDataLoader; import com.kingsrook.qqq.backend.core.instances.loaders.QMetaDataLoaderException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -53,12 +54,12 @@ public class GenericMetaDataLoader extends AbstractMe ** ***************************************************************************/ @Override - public T mapToMetaDataObject(QInstance qInstance, Map map) throws QMetaDataLoaderException + public T mapToMetaDataObject(QInstance qInstance, Map map, LoadingContext context) throws QMetaDataLoaderException { try { T object = metaDataClass.getConstructor().newInstance(); - reflectivelyMap(qInstance, object, map); + reflectivelyMap(qInstance, object, map, context); return (object); } catch(Exception e) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/implementations/QTableMetaDataLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/implementations/QTableMetaDataLoader.java index 963fe93b..d6fa783c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/implementations/QTableMetaDataLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/implementations/QTableMetaDataLoader.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.instances.loaders.implementations; import java.util.Map; +import com.kingsrook.qqq.backend.core.instances.loaders.LoadingContext; import com.kingsrook.qqq.backend.core.instances.loaders.AbstractMetaDataLoader; import com.kingsrook.qqq.backend.core.instances.loaders.QMetaDataLoaderException; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -43,11 +44,11 @@ public class QTableMetaDataLoader extends AbstractMetaDataLoader ** ***************************************************************************/ @Override - public QTableMetaData mapToMetaDataObject(QInstance qInstance, Map map) throws QMetaDataLoaderException + public QTableMetaData mapToMetaDataObject(QInstance qInstance, Map map, LoadingContext context) throws QMetaDataLoaderException { QTableMetaData table = new QTableMetaData(); - reflectivelyMap(qInstance, table, map); + reflectivelyMap(qInstance, table, map, context); // todo - handle QTableBackendDetails, based on backend's type