QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-17 20:50:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Checkpoint on loaders

Browse Source
This commit is contained in:
darin.kelkhoff
2025-01-23 09:39:31 -06:00
parent ea40197893
commit 93c7fbca25
7 changed files with 217 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/AbstractMetaDataLoader.java
View File

@ -31,20 +31,23 @@ import java.lang.reflect.Type;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.core.type.TypeReference;
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QMetaDataObject;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.YamlUtils;
import org.apache.commons.io.IOUtils;
import static com.kingsrook.qqq.backend.core.utils.ValueUtils.getValueAsBoolean;
import static com.kingsrook.qqq.backend.core.utils.ValueUtils.getValueAsInteger;
import static com.kingsrook.qqq.backend.core.utils.ValueUtils.getValueAsString;
@ -59,6 +62,8 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
private String fileName;
private List<LoadingProblem> problems = new ArrayList<>();
/***************************************************************************
@ -68,7 +73,8 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
{
this.fileName = fileName;
Map<String, Object> map = fileToMap(inputStream, fileName);
return (mapToMetaDataObject(qInstance, map));
LoadingContext loadingContext = new LoadingContext(fileName, "/");
return (mapToMetaDataObject(qInstance, map, loadingContext));
}
@ -76,7 +82,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
/***************************************************************************
**
***************************************************************************/
public abstract T mapToMetaDataObject(QInstance qInstance, Map<String, Object> map) throws QMetaDataLoaderException;
public abstract T mapToMetaDataObject(QInstance qInstance, Map<String, Object> map, LoadingContext context) throws QMetaDataLoaderException;
@ -111,9 +117,10 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
/***************************************************************************
*
***************************************************************************/
protected void reflectivelyMap(QInstance qInstance, QMetaDataObject targetObject, Map<String, Object> map)
protected void reflectivelyMap(QInstance qInstance, QMetaDataObject targetObject, Map<String, Object> map, LoadingContext context)
{
Class<? extends QMetaDataObject> targetClass = targetObject.getClass();
Class<? extends QMetaDataObject> targetClass = targetObject.getClass();
Set<String> usedFieldNames = new HashSet<>();
for(Method method : targetClass.getMethods())
{
@ -125,12 +132,13 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
if(map.containsKey(propertyName))
{
usedFieldNames.add(propertyName);
Class<?> parameterType = method.getParameterTypes()[0];
Object rawValue = map.get(propertyName);
try
{
Object mappedValue = reflectivelyMapValue(qInstance, method, parameterType, rawValue);
Object mappedValue = reflectivelyMapValue(qInstance, method, parameterType, rawValue, context.descendToProperty(propertyName));
method.invoke(targetObject, mappedValue);
}
catch(NoValueException nve)
@ -138,15 +146,30 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
///////////////////////
// don't call setter //
///////////////////////
LOG.debug("at " + context + ": No value was mapped for property [" + propertyName + "] on " + targetClass.getSimpleName() + "." + method.getName() + ", raw value: [" + rawValue + "]");
}
}
}
}
catch(Exception e)
{
LOG.warn("Error reflectively mapping on " + targetClass.getName() + "." + method.getName(), e);
addProblem(new LoadingProblem(context, "Error reflectively mapping on " + targetClass.getName() + "." + method.getName(), e));
}
}
//////////////////////////
// mmm, slightly sus... //
//////////////////////////
map.remove("class");
map.remove("version");
Set<String> unrecognizedKeys = new HashSet<>(map.keySet());
unrecognizedKeys.removeAll(usedFieldNames);
if(!unrecognizedKeys.isEmpty())
{
addProblem(new LoadingProblem(context, unrecognizedKeys.size() + " Unrecognized " + StringUtils.plural(unrecognizedKeys, "property", "properties") + ": " + unrecognizedKeys));
}
}
@ -154,26 +177,63 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
/***************************************************************************
*
***************************************************************************/
public Object reflectivelyMapValue(QInstance qInstance, Method method, Class<?> parameterType, Object rawValue) throws Exception
public Object reflectivelyMapValue(QInstance qInstance, Method method, Class<?> parameterType, Object rawValue, LoadingContext context) throws Exception
{
if(rawValue instanceof String s && s.matches("^\\$\\{.+\\..+}"))
{
rawValue = new QMetaDataVariableInterpreter().interpret(s);
LOG.debug("Interpreted raw value [" + s + "] as [" + StringUtils.maskAndTruncate(ValueUtils.getValueAsString(rawValue) + "]"));
}
if(parameterType.equals(String.class))
{
return (getValueAsString(rawValue));
}
else if(parameterType.equals(Integer.class))
{
return (getValueAsInteger(rawValue));
try
{
return (getValueAsInteger(rawValue));
}
catch(Exception e)
{
addProblem(new LoadingProblem(context, "[" + rawValue + "] is not an Integer value."));
}
}
else if(parameterType.equals(Boolean.class))
{
return (getValueAsBoolean(rawValue));
if("true".equals(rawValue) || Boolean.TRUE.equals(rawValue))
{
return (true);
}
else if("false".equals(rawValue) || Boolean.FALSE.equals(rawValue))
{
return (false);
}
else if(rawValue == null)
{
return (null);
}
else
{
addProblem(new LoadingProblem(context, "[" + rawValue + "] is not a boolean value (must be 'true' or 'false')."));
return (null);
}
}
else if(parameterType.equals(boolean.class))
{
Boolean valueAsBoolean = getValueAsBoolean(rawValue);
if(valueAsBoolean != null)
if("true".equals(rawValue) || Boolean.TRUE.equals(rawValue))
{
return (valueAsBoolean);
return (true);
}
else if("false".equals(rawValue) || Boolean.FALSE.equals(rawValue))
{
return (false);
}
else
{
addProblem(new LoadingProblem(context, rawValue + " is not a boolean value (must be 'true' or 'false')."));
throw (new NoValueException());
}
}
else if(parameterType.equals(List.class))
@ -188,7 +248,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
{
try
{
Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, o);
Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, o, context);
mappedValueList.add(mappedValue);
}
catch(NoValueException nve)
@ -211,7 +271,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
{
try
{
Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, o);
Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, o, context);
mappedValueSet.add(mappedValue);
}
catch(NoValueException nve)
@ -227,7 +287,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
Type keyType = ((ParameterizedType) method.getGenericParameterTypes()[0]).getActualTypeArguments()[0];
if(!keyType.equals(String.class))
{
LOG.warn("Unsupported key type for " + method + " (" + keyType + ")");
addProblem(new LoadingProblem(context, "Unsupported key type for " + method + " got [" + keyType + "], expected [String]"));
throw new NoValueException();
}
// todo make sure string
@ -244,7 +304,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
{
@SuppressWarnings("unchecked")
Map.Entry<String, Object> entry = (Map.Entry<String, Object>) o;
Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, entry.getValue());
Object mappedValue = reflectivelyMapValue(qInstance, null, actualTypeClass, entry.getValue(), context);
mappedValueMap.put(entry.getKey(), mappedValue);
}
catch(NoValueException nve)
@ -265,6 +325,8 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
return (enumConstant);
}
}
addProblem(new LoadingProblem(context, "Unrecognized value [" + rawValue + "]. Expected one of: " + Arrays.toString(parameterType.getEnumConstants())));
}
else if(MetaDataLoaderRegistry.hasLoaderForClass(parameterType))
{
@ -273,7 +335,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
Class<? extends AbstractMetaDataLoader<?>> loaderClass = MetaDataLoaderRegistry.getLoaderForClass(parameterType);
AbstractMetaDataLoader<?> loader = loaderClass.getConstructor().newInstance();
//noinspection unchecked
return (loader.mapToMetaDataObject(qInstance, valueMap));
return (loader.mapToMetaDataObject(qInstance, valueMap, context));
}
}
else if(QMetaDataObject.class.isAssignableFrom(parameterType))
@ -282,7 +344,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
{
QMetaDataObject childObject = (QMetaDataObject) parameterType.getConstructor().newInstance();
//noinspection unchecked
reflectivelyMap(qInstance, childObject, valueMap);
reflectivelyMap(qInstance, childObject, valueMap, context);
return (childObject);
}
}
@ -300,7 +362,7 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
else
{
// todo clean up this message/level
LOG.warn("No case for " + parameterType + " (arg to: " + method + ")");
addProblem(new LoadingProblem(context, "No case for " + parameterType + " (arg to: " + method + ")"));
}
throw new NoValueException();
@ -424,4 +486,25 @@ public abstract class AbstractMetaDataLoader<T extends QMetaDataObject>
super("No value");
}
}
/***************************************************************************
**
***************************************************************************/
public void addProblem(LoadingProblem problem)
{
problems.add(problem);
}
/*******************************************************************************
** Getter for problems
**
*******************************************************************************/
public List<LoadingProblem> getProblems()
{
return (problems);
}
}

19
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/ClassDetectingMetaDataLoader.java
View File

@ -25,11 +25,14 @@ package com.kingsrook.qqq.backend.core.instances.loaders;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.instances.loaders.implementations.GenericMetaDataLoader;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QMetaDataObject;
import com.kingsrook.qqq.backend.core.utils.ClassPathUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.AnyKey;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
@ -39,6 +42,8 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
*******************************************************************************/
public class ClassDetectingMetaDataLoader extends AbstractMetaDataLoader<QMetaDataObject>
{
private static final Memoization<AnyKey, List<Class<?>>> memoizedMetaDataObjectClasses = new Memoization<>();
/***************************************************************************
*
@ -61,7 +66,6 @@ public class ClassDetectingMetaDataLoader extends AbstractMetaDataLoader<QMetaDa
String classProperty = ValueUtils.getValueAsString(map.get("class"));
try
{
if(MetaDataLoaderRegistry.hasLoaderForSimpleName(classProperty))
{
Class<? extends AbstractMetaDataLoader<?>> loaderClass = MetaDataLoaderRegistry.getLoaderForSimpleName(classProperty);
@ -69,8 +73,13 @@ public class ClassDetectingMetaDataLoader extends AbstractMetaDataLoader<QMetaDa
}
else
{
List<Class<?>> classesInPackage = ClassPathUtils.getClassesInPackage("com.kingsrook.qqq.backend.core.model");
for(Class<?> c : classesInPackage)
Optional<List<Class<?>>> metaDataClasses = memoizedMetaDataObjectClasses.getResult(AnyKey.getInstance(), k -> ClassPathUtils.getClassesContainingNameAndOfType("MetaData", QMetaDataObject.class));
if(metaDataClasses.isEmpty())
{
throw (new QMetaDataLoaderException("Could not get list of metaDataObjects from class loader"));
}
for(Class<?> c : metaDataClasses.get())
{
if(c.getSimpleName().equals(classProperty) && QMetaDataObject.class.isAssignableFrom(c))
{
@ -103,9 +112,9 @@ public class ClassDetectingMetaDataLoader extends AbstractMetaDataLoader<QMetaDa
**
***************************************************************************/
@Override
public QMetaDataObject mapToMetaDataObject(QInstance qInstance, Map<String, Object> map) throws QMetaDataLoaderException
public QMetaDataObject mapToMetaDataObject(QInstance qInstance, Map<String, Object> map, LoadingContext context) throws QMetaDataLoaderException
{
AbstractMetaDataLoader<?> loaderForMap = getLoaderForMap(map);
return loaderForMap.mapToMetaDataObject(qInstance, map);
return loaderForMap.mapToMetaDataObject(qInstance, map, context);
}
}

38
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/LoadingContext.java Normal file
View File

@ -0,0 +1,38 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.instances.loaders;
/*******************************************************************************
** Record to track where loader objects are - e.g., what file they're on,
** and at what property path within the file (e.g., helps report problems).
*******************************************************************************/
public record LoadingContext(String fileName, String propertyPath)
{
/***************************************************************************
**
***************************************************************************/
public LoadingContext descendToProperty(String propertyName)
{
return new LoadingContext(fileName, propertyPath + propertyName + "/");
}
}

49
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/LoadingProblem.java Normal file
View File

@ -0,0 +1,49 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.instances.loaders;
/*******************************************************************************
** record that tracks a problem that was encountered when loading files.
*******************************************************************************/
public record LoadingProblem(LoadingContext context, String message, Exception exception) // todo Level if useful
{
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public LoadingProblem(LoadingContext context, String message)
{
this(context, message, null);
}
/***************************************************************************
**
***************************************************************************/
@Override
public String toString()
{
return "at[" + context.fileName() + "][" + context.propertyPath() + "]: " + message;
}
}

7
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/MetaDataLoaderHelper.java
View File

@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QMetaDataObject;
import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
@ -64,6 +65,12 @@ public class MetaDataLoaderHelper
try(FileInputStream fileInputStream = new FileInputStream(file))
{
QMetaDataObject qMetaDataObject = loader.fileToMetaDataObject(qInstance, fileInputStream, file.getName());
if(CollectionUtils.nullSafeHasContents(loader.getProblems()))
{
loader.getProblems().forEach(System.out::println);
}
if(qMetaDataObject instanceof TopLevelMetaDataInterface topLevelMetaData)
{
topLevelMetaData.addSelfToInstance(qInstance);

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/implementations/GenericMetaDataLoader.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.instances.loaders.implementations;
import java.util.Map;
import com.kingsrook.qqq.backend.core.instances.loaders.LoadingContext;
import com.kingsrook.qqq.backend.core.instances.loaders.AbstractMetaDataLoader;
import com.kingsrook.qqq.backend.core.instances.loaders.QMetaDataLoaderException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -53,12 +54,12 @@ public class GenericMetaDataLoader<T extends QMetaDataObject> extends AbstractMe
**
***************************************************************************/
@Override
public T mapToMetaDataObject(QInstance qInstance, Map<String, Object> map) throws QMetaDataLoaderException
public T mapToMetaDataObject(QInstance qInstance, Map<String, Object> map, LoadingContext context) throws QMetaDataLoaderException
{
try
{
T object = metaDataClass.getConstructor().newInstance();
reflectivelyMap(qInstance, object, map);
reflectivelyMap(qInstance, object, map, context);
return (object);
}
catch(Exception e)

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/loaders/implementations/QTableMetaDataLoader.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.instances.loaders.implementations;
import java.util.Map;
import com.kingsrook.qqq.backend.core.instances.loaders.LoadingContext;
import com.kingsrook.qqq.backend.core.instances.loaders.AbstractMetaDataLoader;
import com.kingsrook.qqq.backend.core.instances.loaders.QMetaDataLoaderException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
@ -43,11 +44,11 @@ public class QTableMetaDataLoader extends AbstractMetaDataLoader<QTableMetaData>
**
***************************************************************************/
@Override
public QTableMetaData mapToMetaDataObject(QInstance qInstance, Map<String, Object> map) throws QMetaDataLoaderException
public QTableMetaData mapToMetaDataObject(QInstance qInstance, Map<String, Object> map, LoadingContext context) throws QMetaDataLoaderException
{
QTableMetaData table = new QTableMetaData();
reflectivelyMap(qInstance, table, map);
reflectivelyMap(qInstance, table, map, context);
// todo - handle QTableBackendDetails, based on backend's type