From 89a943bc2cad9f3c4774265a0c29d1cfe9a9694b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 1 Aug 2022 19:43:59 -0500 Subject: [PATCH 01/18] more entity->QRecord abilities; add TIME & BOOLEAN types --- .../core/instances/QInstanceEnricher.java | 109 +++++++++++++++ .../actions/tables/query/QFilterCriteria.java | 26 +++- .../qqq/backend/core/model/data/QField.java | 54 ++++++++ .../core/model/data/QRecordEntity.java | 43 +++++- .../core/model/data/QRecordEntityField.java | 76 +++++++--- .../model/metadata/fields/QFieldMetaData.java | 45 +++++- .../model/metadata/fields/QFieldType.java | 22 ++- .../model/metadata/tables/QTableMetaData.java | 31 +++++ .../qqq/backend/core/utils/ListingHash.java | 16 ++- .../qqq/backend/core/utils/ValueUtils.java | 119 ++++++++++++++-- .../actions/metadata/MetaDataActionTest.java | 14 +- .../processes/RunBackendStepActionTest.java | 3 +- .../core/instances/QInstanceEnricherTest.java | 65 ++++++++- .../core/model/data/QRecordEntityTest.java | 63 ++++++++- .../core/model/data/testentities/Item.java | 17 ++- .../qqq/backend/core/utils/TestUtils.java | 48 ++++--- .../backend/core/utils/ValueUtilsTest.java | 86 ++++++++++++ .../rdbms/actions/AbstractRDBMSAction.java | 16 ++- .../rdbms/actions/RDBMSQueryAction.java | 5 + .../module/rdbms/jdbc/QueryManager.java | 80 ++++++++++- .../module/rdbms/jdbc/QueryManagerTest.java | 131 +++++++++++++----- .../qqq/frontend/picocli/QCommandBuilder.java | 4 +- 22 files changed, 956 insertions(+), 117 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 569ce603..b69a1da9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -47,7 +47,10 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEd import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertStoreRecordsStep; import com.kingsrook.qqq.backend.core.processes.implementations.general.LoadInitialRecordsStep; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -57,6 +60,10 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class QInstanceEnricher { + private static final Logger LOG = LogManager.getLogger(QInstanceEnricher.class); + + + /******************************************************************************* ** *******************************************************************************/ @@ -182,6 +189,16 @@ public class QInstanceEnricher return (null); } + if(name.length() == 0) + { + return (""); + } + + if(name.length() == 1) + { + return (name.substring(0, 1).toUpperCase(Locale.ROOT)); + } + return (name.substring(0, 1).toUpperCase(Locale.ROOT) + name.substring(1).replaceAll("([A-Z])", " $1")); } @@ -403,4 +420,96 @@ public class QInstanceEnricher ))); } + + + /******************************************************************************* + ** for all fields in a table, set their backendName, using the default "inference" logic + ** see {@link #inferBackendName(String)} + *******************************************************************************/ + public static void setInferredFieldBackendNames(QTableMetaData tableMetaData) + { + if(tableMetaData == null) + { + LOG.warn("Requested to infer field backend names with a null table as input. Returning with noop."); + return; + } + + if(CollectionUtils.nullSafeIsEmpty(tableMetaData.getFields())) + { + LOG.warn("Requested to infer field backend names on a table [" + tableMetaData.getName() + "] with no fields. Returning with noop."); + return; + } + + for(QFieldMetaData field : tableMetaData.getFields().values()) + { + String fieldName = field.getName(); + String fieldBackendName = field.getBackendName(); + if(!StringUtils.hasContent(fieldBackendName)) + { + String backendName = inferBackendName(fieldName); + field.setBackendName(backendName); + } + } + } + + + + /******************************************************************************* + ** Do a default mapping from a camelCase field name to an underscore_style + ** name for a backend. + *******************************************************************************/ + static String inferBackendName(String fieldName) + { + //////////////////////////////////////////////////////////////////////////////////////// + // build a list of words in the name, then join them with _ and lower-case the result // + //////////////////////////////////////////////////////////////////////////////////////// + List words = new ArrayList<>(); + StringBuilder currentWord = new StringBuilder(); + for(int i = 0; i < fieldName.length(); i++) + { + Character thisChar = fieldName.charAt(i); + Character nextChar = i < (fieldName.length() - 1) ? fieldName.charAt(i + 1) : null; + + ///////////////////////////////////////////////////////////////////////////////////// + // if we're at the end of the whole string, then we're at the end of the last word // + ///////////////////////////////////////////////////////////////////////////////////// + if(nextChar == null) + { + currentWord.append(thisChar); + words.add(currentWord.toString()); + } + + /////////////////////////////////////////////////////////// + // transitioning from a lower to an upper starts a word. // + /////////////////////////////////////////////////////////// + else if(Character.isLowerCase(thisChar) && Character.isUpperCase(nextChar)) + { + currentWord.append(thisChar); + words.add(currentWord.toString()); + currentWord = new StringBuilder(); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // transitioning from an upper to a lower - it starts a word, as long as there were already letters in the current word // + // e.g., on wordThenTLAInMiddle, when thisChar=I and nextChar=n. currentWord will be "TLA". So finish that word, and start a new one with the 'I' // + // but the normal single-upper condition, e.g., firstName, when thisChar=N and nextChar=a, current word will be empty string, so just append the 'a' to it // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + else if(Character.isUpperCase(thisChar) && Character.isLowerCase(nextChar) && currentWord.length() > 0) + { + words.add(currentWord.toString()); + currentWord = new StringBuilder(); + currentWord.append(thisChar); + } + + ///////////////////////////////////////////////////////////// + // by default, just add this character to the current word // + ///////////////////////////////////////////////////////////// + else + { + currentWord.append(thisChar); + } + } + + return (String.join("_", words).toLowerCase(Locale.ROOT)); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java index d4c5959b..251a961b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java @@ -32,12 +32,33 @@ import java.util.List; *******************************************************************************/ public class QFilterCriteria implements Serializable { - private String fieldName; - private QCriteriaOperator operator; + private String fieldName; + private QCriteriaOperator operator; private List values; + /******************************************************************************* + ** + *******************************************************************************/ + public QFilterCriteria() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QFilterCriteria(String fieldName, QCriteriaOperator operator, List values) + { + this.fieldName = fieldName; + this.operator = operator; + this.values = values; + } + + + /******************************************************************************* ** Getter for fieldName ** @@ -127,6 +148,7 @@ public class QFilterCriteria implements Serializable } + /******************************************************************************* ** Setter for values ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java new file mode 100644 index 00000000..cbdb9616 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java @@ -0,0 +1,54 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.data; + + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; + + +/******************************************************************************* + ** + *******************************************************************************/ +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.RUNTIME) +public @interface QField +{ + /******************************************************************************* + ** + *******************************************************************************/ + String backendName() default ""; + + /******************************************************************************* + ** + *******************************************************************************/ + boolean isRequired() default false; + + /******************************************************************************* + ** + *******************************************************************************/ + boolean isEditable() default true; +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java index df5c8ecc..2db368ba 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java @@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.model.data; import java.io.Serializable; +import java.lang.reflect.Field; import java.lang.reflect.Method; import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -104,7 +108,7 @@ public abstract class QRecordEntity /******************************************************************************* ** *******************************************************************************/ - private static List getFieldList(Class c) + public static List getFieldList(Class c) { if(!fieldMapping.containsKey(c)) { @@ -114,10 +118,12 @@ public abstract class QRecordEntity if(isGetter(possibleGetter)) { Optional setter = getSetterForGetter(c, possibleGetter); + if(setter.isPresent()) { - String name = getFieldNameFromGetter(possibleGetter); - fieldList.add(new QRecordEntityField(name, possibleGetter, setter.get(), possibleGetter.getReturnType())); + String fieldName = getFieldNameFromGetter(possibleGetter); + Optional fieldAnnotation = getQFieldAnnotation(c, fieldName); + fieldList.add(new QRecordEntityField(fieldName, possibleGetter, setter.get(), possibleGetter.getReturnType(), fieldAnnotation.orElse(null))); } else { @@ -132,6 +138,27 @@ public abstract class QRecordEntity + /******************************************************************************* + ** + *******************************************************************************/ + public static Optional getQFieldAnnotation(Class c, String fieldName) + { + try + { + Field field = c.getDeclaredField(fieldName); + return (Optional.ofNullable(field.getAnnotation(QField.class))); + } + catch(NoSuchFieldException e) + { + ////////////////////////////////////////// + // ok, we just won't have an annotation // + ////////////////////////////////////////// + } + return (Optional.empty()); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -207,7 +234,15 @@ public abstract class QRecordEntity || returnType.equals(int.class) || returnType.equals(Boolean.class) || returnType.equals(boolean.class) - || returnType.equals(BigDecimal.class)); + || returnType.equals(BigDecimal.class) + || returnType.equals(Instant.class) + || returnType.equals(LocalDate.class) + || returnType.equals(LocalTime.class)); + ///////////////////////////////////////////// + // note - this list has implications upon: // + // - QFieldType.fromClass // + // - QRecordEntityField.convertValueType // + ///////////////////////////////////////////// } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java index b03dbea6..f977f244 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java @@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.model.data; import java.io.Serializable; import java.lang.reflect.Method; import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; import com.kingsrook.qqq.backend.core.exceptions.QValueException; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -38,18 +40,20 @@ public class QRecordEntityField private final Method getter; private final Method setter; private final Class type; + private final QField fieldAnnotation; /******************************************************************************* ** Constructor. *******************************************************************************/ - public QRecordEntityField(String fieldName, Method getter, Method setter, Class type) + public QRecordEntityField(String fieldName, Method getter, Method setter, Class type, QField fieldAnnotation) { this.fieldName = fieldName; this.getter = getter; this.setter = setter; this.type = type; + this.fieldAnnotation = fieldAnnotation; } @@ -98,39 +102,67 @@ public class QRecordEntityField + /******************************************************************************* + ** Getter for fieldAnnotation + ** + *******************************************************************************/ + public QField getFieldAnnotation() + { + return fieldAnnotation; + } + + + /******************************************************************************* ** *******************************************************************************/ public Object convertValueType(Serializable value) { - if(value == null) + try { - return (null); - } + if(value == null) + { + return (null); + } - if(value.getClass().equals(type)) - { - return (value); - } + if(value.getClass().equals(type)) + { + return (value); + } - if(type.equals(String.class)) - { - return (ValueUtils.getValueAsString(value)); - } + if(type.equals(String.class)) + { + return (ValueUtils.getValueAsString(value)); + } - if(type.equals(Integer.class) || type.equals(int.class)) - { - return (ValueUtils.getValueAsInteger(value)); - } + if(type.equals(Integer.class) || type.equals(int.class)) + { + return (ValueUtils.getValueAsInteger(value)); + } - if(type.equals(Boolean.class) || type.equals(boolean.class)) - { - return (ValueUtils.getValueAsBoolean(value)); - } + if(type.equals(Boolean.class) || type.equals(boolean.class)) + { + return (ValueUtils.getValueAsBoolean(value)); + } - if(type.equals(BigDecimal.class)) + if(type.equals(BigDecimal.class)) + { + return (ValueUtils.getValueAsBigDecimal(value)); + } + + if(type.equals(LocalDate.class)) + { + return (ValueUtils.getValueAsLocalDate(value)); + } + + if(type.equals(Instant.class)) + { + return (ValueUtils.getValueAsInstant(value)); + } + } + catch(Exception e) { - return (ValueUtils.getValueAsBigDecimal(value)); + throw (new QValueException("Exception converting value [" + value + "] for field [" + fieldName + "]", e)); } throw (new QValueException("Unhandled value type [" + type + "] for field [" + fieldName + "]")); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java index c5fb9e72..99c0fbf3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java @@ -24,9 +24,13 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields; import java.io.Serializable; import java.lang.reflect.Method; +import java.util.Optional; import com.github.hervian.reflection.Fun; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -77,12 +81,49 @@ public class QFieldMetaData ** e.g., new QFieldMetaData(Order::getOrderNo). *******************************************************************************/ public QFieldMetaData(Fun.With1ParamAndVoid getterRef) throws QException + { + Method getter = Fun.toMethod(getterRef); + constructFromGetter(getter); + } + + + + /******************************************************************************* + ** Initialize a fieldMetaData from a getter method from an entity + ** + *******************************************************************************/ + public QFieldMetaData(Method getter) throws QException + { + constructFromGetter(getter); + } + + + + /******************************************************************************* + ** From a getter method, populate attributes in this field meta-data, including + ** those from the @QField annotation on the field in the class, if present. + *******************************************************************************/ + private void constructFromGetter(Method getter) throws QException { try { - Method getter = Fun.toMethod(getterRef); this.name = QRecordEntity.getFieldNameFromGetter(getter); this.type = QFieldType.fromClass(getter.getReturnType()); + + @SuppressWarnings("unchecked") + Optional optionalFieldAnnotation = QRecordEntity.getQFieldAnnotation((Class) getter.getDeclaringClass(), this.name); + + if(optionalFieldAnnotation.isPresent()) + { + QField fieldAnnotation = optionalFieldAnnotation.get(); + setIsRequired(fieldAnnotation.isRequired()); + setIsEditable(fieldAnnotation.isEditable()); + + if(StringUtils.hasContent(fieldAnnotation.backendName())) + { + setBackendName(fieldAnnotation.backendName()); + } + } } catch(QException qe) { @@ -90,7 +131,7 @@ public class QFieldMetaData } catch(Exception e) { - throw (new QException("Error constructing field from getterRef: " + getterRef, e)); + throw (new QException("Error constructing field from getter method: " + getter.getName(), e)); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java index 324e8991..41ffdcc4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java @@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields; import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -35,8 +38,9 @@ public enum QFieldType STRING, INTEGER, DECIMAL, + BOOLEAN, DATE, - // TIME, + TIME, DATE_TIME, TEXT, HTML, @@ -65,6 +69,22 @@ public enum QFieldType { return (DECIMAL); } + if(c.equals(Instant.class)) + { + return (DATE_TIME); + } + if(c.equals(LocalDate.class)) + { + return (DATE); + } + if(c.equals(LocalTime.class)) + { + return (TIME); + } + if(c.equals(Boolean.class)) + { + return (BOOLEAN); + } throw (new QException("Unrecognized class [" + c + "]")); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index aed8f51c..49695979 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -28,6 +28,10 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntityField; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -91,6 +95,22 @@ public class QTableMetaData implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData withFieldsFromEntity(Class entityClass) throws QException + { + List recordEntityFieldList = QRecordEntity.getFieldList(entityClass); + for(QRecordEntityField recordEntityField : recordEntityFieldList) + { + QFieldMetaData field = new QFieldMetaData(recordEntityField.getGetter()); + addField(field); + } + return (this); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -434,4 +454,15 @@ public class QTableMetaData implements Serializable return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData withInferredFieldBackendNames() + { + QInstanceEnricher.setInferredFieldBackendNames(this); + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ListingHash.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ListingHash.java index 58eefe68..ebb35e2a 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ListingHash.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ListingHash.java @@ -41,7 +41,7 @@ public class ListingHash implements Map>, Serializable { public static final long serialVersionUID = 0L; - private HashMap> hashMap = null; + private Map> hashMap = null; @@ -51,7 +51,19 @@ public class ListingHash implements Map>, Serializable *******************************************************************************/ public ListingHash() { - this.hashMap = new HashMap>(); + this.hashMap = new HashMap<>(); + } + + + + /******************************************************************************* + ** Constructor where you can supply a source map (e.g., if you want a specific + ** Map type (like LinkedHashMap), or with pre-values + ** + *******************************************************************************/ + public ListingHash(Map> sourceMap) + { + this.hashMap = sourceMap; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index 652273fa..d839a36b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -23,11 +23,14 @@ package com.kingsrook.qqq.backend.core.utils; import java.math.BigDecimal; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.Calendar; +import java.util.List; import java.util.TimeZone; import com.kingsrook.qqq.backend.core.exceptions.QValueException; @@ -37,7 +40,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QValueException; *******************************************************************************/ public class ValueUtils { - private static final DateTimeFormatter localDateDefaultFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + private static final DateTimeFormatter yyyyMMddWithDashesFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + private static final DateTimeFormatter MdyyyyWithSlashesFormatter = DateTimeFormatter.ofPattern("M/d/yyyy"); @@ -174,9 +178,13 @@ public class ValueUtils } else { - throw (new IllegalArgumentException("Unsupported class " + value.getClass().getName() + " for converting to Integer.")); + throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to Integer.")); } } + catch(QValueException qve) + { + throw (qve); + } catch(Exception e) { throw (new QValueException("Value [" + value + "] could not be converted to an Integer.", e)); @@ -212,8 +220,8 @@ public class ValueUtils } else if(value instanceof Calendar c) { - TimeZone tz = c.getTimeZone(); - ZoneId zid = (tz == null) ? ZoneId.systemDefault() : tz.toZoneId(); + TimeZone tz = c.getTimeZone(); + ZoneId zid = (tz == null) ? ZoneId.systemDefault() : tz.toZoneId(); return LocalDateTime.ofInstant(c.toInstant(), zid).toLocalDate(); } else if(value instanceof LocalDateTime ldt) @@ -227,21 +235,47 @@ public class ValueUtils return (null); } - return LocalDate.parse(s, localDateDefaultFormatter); + return tryLocalDateParsers(s); } else { - throw (new IllegalArgumentException("Unsupported class " + value.getClass().getName() + " for converting to LocalDate.")); + throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to LocalDate.")); } } + catch(QValueException qve) + { + throw (qve); + } catch(Exception e) { - throw (new QValueException("Value [" + value + "] could not be converted to an LocalDate.", e)); + throw (new QValueException("Value [" + value + "] could not be converted to a LocalDate.", e)); } } + /******************************************************************************* + ** + *******************************************************************************/ + private static LocalDate tryLocalDateParsers(String s) + { + DateTimeParseException lastException = null; + for(DateTimeFormatter dateTimeFormatter : List.of(yyyyMMddWithDashesFormatter, MdyyyyWithSlashesFormatter)) + { + try + { + return LocalDate.parse(s, dateTimeFormatter); + } + catch(DateTimeParseException dtpe) + { + lastException = dtpe; + } + } + throw (new QValueException("Could not parse value [" + s + "] to a local date", lastException)); + } + + + /******************************************************************************* ** Type-safely make a BigDecimal from any Object. ** null and empty-string inputs return null. @@ -305,13 +339,80 @@ public class ValueUtils } else { - throw (new IllegalArgumentException("Unsupported class " + value.getClass().getName() + " for converting to BigDecimal.")); + throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to BigDecimal.")); } } + catch(QValueException qve) + { + throw (qve); + } catch(Exception e) { - throw (new QValueException("Value [" + value + "] could not be converted to an BigDecimal.", e)); + throw (new QValueException("Value [" + value + "] could not be converted to a BigDecimal.", e)); } } + + + /******************************************************************************* + ** Type-safely make an Instant from any Object. + ** null and empty-string inputs return null. + ** We may throw if the input can't be converted to a Instant + *******************************************************************************/ + public static Instant getValueAsInstant(Object value) + { + try + { + if(value == null) + { + return (null); + } + else if(value instanceof Instant i) + { + return (i); + } + else if(value instanceof java.sql.Date d) + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // note - in the jdk, this method throws UnsupportedOperationException (because of the lack of time in sql Dates) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + return d.toInstant(); + } + else if(value instanceof java.util.Date d) + { + return d.toInstant(); + } + else if(value instanceof Calendar c) + { + TimeZone tz = c.getTimeZone(); + return (c.toInstant()); + } + else if(value instanceof LocalDateTime ldt) + { + ZoneId zoneId = ZoneId.systemDefault(); + return ldt.toInstant(zoneId.getRules().getOffset(ldt)); + } + else if(value instanceof String s) + { + if(!StringUtils.hasContent(s)) + { + return (null); + } + + return Instant.parse(s); + } + else + { + throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to Instant.")); + } + } + catch(QValueException qve) + { + throw (qve); + } + catch(Exception e) + { + throw (new QValueException("Value [" + value + "] could not be converted to a Instant.", e)); + } + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java index f41b7012..f66f81ea 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java @@ -22,12 +22,15 @@ package com.kingsrook.qqq.backend.core.actions.metadata; +import java.util.Set; +import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; /******************************************************************************* @@ -47,9 +50,18 @@ class MetaDataActionTest request.setSession(TestUtils.getMockSession()); MetaDataOutput result = new MetaDataAction().execute(request); assertNotNull(result); + assertNotNull(result.getTables()); assertNotNull(result.getTables().get("person")); assertEquals("person", result.getTables().get("person").getName()); assertEquals("Person", result.getTables().get("person").getLabel()); + + assertNotNull(result.getProcesses().get("greet")); + assertNotNull(result.getProcesses().get("greetInteractive")); + assertNotNull(result.getProcesses().get("etl.basic")); + assertNotNull(result.getProcesses().get("person.bulkInsert")); + assertNotNull(result.getProcesses().get("person.bulkEdit")); + assertNotNull(result.getProcesses().get("person.bulkDelete")); + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java index 3b88e502..c1d308dc 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepActionTest.java @@ -103,7 +103,8 @@ public class RunBackendStepActionTest case STRING -> "ABC"; case INTEGER -> 42; case DECIMAL -> new BigDecimal("47"); - case DATE, DATE_TIME -> null; + case BOOLEAN -> true; + case DATE, TIME, DATE_TIME -> null; case TEXT -> """ ABC XYZ"""; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java index 0e6ef951..f9017298 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java @@ -22,8 +22,11 @@ package com.kingsrook.qqq.backend.core.instances; +import java.util.Collections; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; @@ -45,7 +48,7 @@ class QInstanceEnricherTest @Test public void test_nullTableLabelComesFromName() { - QInstance qInstance = TestUtils.defineInstance(); + QInstance qInstance = TestUtils.defineInstance(); QTableMetaData personTable = qInstance.getTable("person"); personTable.setLabel(null); assertNull(personTable.getLabel()); @@ -54,6 +57,7 @@ class QInstanceEnricherTest } + /******************************************************************************* ** Test that a table missing a label and a name doesn't NPE, but just keeps ** the name & label both null. @@ -62,7 +66,7 @@ class QInstanceEnricherTest @Test public void test_nullNameGivesNullLabel() { - QInstance qInstance = TestUtils.defineInstance(); + QInstance qInstance = TestUtils.defineInstance(); QTableMetaData personTable = qInstance.getTable("person"); personTable.setLabel(null); personTable.setName(null); @@ -74,6 +78,7 @@ class QInstanceEnricherTest } + /******************************************************************************* ** Test that a field missing a label gets the default label applied (name w/ UC-first) ** @@ -81,12 +86,64 @@ class QInstanceEnricherTest @Test public void test_nullFieldLabelComesFromName() { - QInstance qInstance = TestUtils.defineInstance(); - QFieldMetaData idField = qInstance.getTable("person").getField("id"); + QInstance qInstance = TestUtils.defineInstance(); + QFieldMetaData idField = qInstance.getTable("person").getField("id"); idField.setLabel(null); assertNull(idField.getLabel()); new QInstanceEnricher().enrich(qInstance); assertEquals("Id", idField.getLabel()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSetInferredFieldBackendNames() + { + QTableMetaData table = new QTableMetaData() + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("firstName", QFieldType.INTEGER)) + .withField(new QFieldMetaData("nonstandard", QFieldType.INTEGER).withBackendName("whateverImNon_standard")); + QInstanceEnricher.setInferredFieldBackendNames(table); + assertEquals("id", table.getField("id").getBackendName()); + assertEquals("first_name", table.getField("firstName").getBackendName()); + assertEquals("whateverImNon_standard", table.getField("nonstandard").getBackendName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSetInferredFieldBackendNamesEdgeCases() + { + /////////////////////////////////////////////////////////////// + // make sure none of these cases throw (but all should warn) // + /////////////////////////////////////////////////////////////// + QInstanceEnricher.setInferredFieldBackendNames(null); + QInstanceEnricher.setInferredFieldBackendNames(new QTableMetaData()); + QInstanceEnricher.setInferredFieldBackendNames(new QTableMetaData().withFields(Collections.emptyMap())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testInferBackendName() + { + assertEquals("id", QInstanceEnricher.inferBackendName("id")); + assertEquals("word_another_word_more_words", QInstanceEnricher.inferBackendName("wordAnotherWordMoreWords")); + assertEquals("l_ul_ul_ul", QInstanceEnricher.inferBackendName("lUlUlUl")); + assertEquals("starts_upper", QInstanceEnricher.inferBackendName("StartsUpper")); + assertEquals("tla_first", QInstanceEnricher.inferBackendName("TLAFirst")); + assertEquals("word_then_tla_in_middle", QInstanceEnricher.inferBackendName("wordThenTLAInMiddle")); + assertEquals("end_with_tla", QInstanceEnricher.inferBackendName("endWithTLA")); + assertEquals("tla_and_another_tla", QInstanceEnricher.inferBackendName("TLAAndAnotherTLA")); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java index e8f8e9e3..be3cd897 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java @@ -162,15 +162,74 @@ class QRecordEntityTest *******************************************************************************/ @SuppressWarnings("ResultOfMethodCallIgnored") @Test - void testQTableConstructionFromEntity() throws QException + void testQTableConstructionFromEntityGetterReferences() throws QException { QTableMetaData qTableMetaData = new QTableMetaData() .withField(new QFieldMetaData(Item::getSku)) .withField(new QFieldMetaData(Item::getDescription)) - .withField(new QFieldMetaData(Item::getQuantity)); + .withField(new QFieldMetaData(Item::getQuantity)) + .withField(new QFieldMetaData(Item::getFeatured)) + .withField(new QFieldMetaData(Item::getPrice)); assertEquals(QFieldType.STRING, qTableMetaData.getField("sku").getType()); assertEquals(QFieldType.INTEGER, qTableMetaData.getField("quantity").getType()); + + /////////////////////////////////////////////////////////////// + // assert about attributes that came from @QField annotation // + /////////////////////////////////////////////////////////////// + assertTrue(qTableMetaData.getField("sku").getIsRequired()); + assertFalse(qTableMetaData.getField("quantity").getIsEditable()); + assertEquals("is_featured", qTableMetaData.getField("featured").getBackendName()); + + ////////////////////////////////////////////////////////////////////////// + // assert about attributes that weren't specified in @QField annotation // + ////////////////////////////////////////////////////////////////////////// + assertTrue(qTableMetaData.getField("sku").getIsEditable()); + assertFalse(qTableMetaData.getField("quantity").getIsRequired()); + assertNull(qTableMetaData.getField("sku").getBackendName()); + + ///////////////////////////////////////////////////////////////////// + // assert about attributes for fields without a @QField annotation // + ///////////////////////////////////////////////////////////////////// + assertTrue(qTableMetaData.getField("price").getIsEditable()); + assertFalse(qTableMetaData.getField("price").getIsRequired()); + assertNull(qTableMetaData.getField("price").getBackendName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQTableConstructionFromEntity() throws QException + { + QTableMetaData qTableMetaData = new QTableMetaData() + .withFieldsFromEntity(Item.class); + + assertEquals(QFieldType.STRING, qTableMetaData.getField("sku").getType()); + assertEquals(QFieldType.INTEGER, qTableMetaData.getField("quantity").getType()); + + /////////////////////////////////////////////////////////////// + // assert about attributes that came from @QField annotation // + /////////////////////////////////////////////////////////////// + assertTrue(qTableMetaData.getField("sku").getIsRequired()); + assertFalse(qTableMetaData.getField("quantity").getIsEditable()); + assertEquals("is_featured", qTableMetaData.getField("featured").getBackendName()); + + ////////////////////////////////////////////////////////////////////////// + // assert about attributes that weren't specified in @QField annotation // + ////////////////////////////////////////////////////////////////////////// + assertTrue(qTableMetaData.getField("sku").getIsEditable()); + assertFalse(qTableMetaData.getField("quantity").getIsRequired()); + assertNull(qTableMetaData.getField("sku").getBackendName()); + + ///////////////////////////////////////////////////////////////////// + // assert about attributes for fields without a @QField annotation // + ///////////////////////////////////////////////////////////////////// + assertTrue(qTableMetaData.getField("price").getIsEditable()); + assertFalse(qTableMetaData.getField("price").getIsRequired()); + assertNull(qTableMetaData.getField("price").getBackendName()); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java index 4412122b..86653880 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.data.testentities; import java.math.BigDecimal; +import com.kingsrook.qqq.backend.core.model.data.QField; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; @@ -31,11 +32,19 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; *******************************************************************************/ public class Item extends QRecordEntity { - private String sku; - private String description; - private Integer quantity; + @QField(isRequired = true) + private String sku; + + @QField() + private String description; + + @QField(isEditable = false) + private Integer quantity; + private BigDecimal price; - private Boolean featured; + + @QField(backendName = "is_featured") + private Boolean featured; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 6bae6380..d7f6b5d0 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -32,16 +32,13 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; -import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; -import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; @@ -50,10 +47,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationModule; +import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; /******************************************************************************* @@ -62,9 +62,15 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicE *******************************************************************************/ public class TestUtils { - public static String DEFAULT_BACKEND_NAME = "default"; - public static String PROCESS_NAME_GREET_PEOPLE = "greet"; - public static String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive"; + public static final String DEFAULT_BACKEND_NAME = "default"; + + public static final String TABLE_NAME_PERSON = "person"; + + public static final String PROCESS_NAME_GREET_PEOPLE = "greet"; + public static final String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive"; + public static final String PROCESS_NAME_ADD_TO_PEOPLES_AGE = "addToPeoplesAge"; + public static final String TABLE_NAME_PERSON_FILE = "personFile"; + public static final String TABLE_NAME_ID_AND_NAME_ONLY = "idAndNameOnly"; @@ -138,7 +144,7 @@ public class TestUtils public static QTableMetaData defineTablePerson() { return new QTableMetaData() - .withName("person") + .withName(TABLE_NAME_PERSON) .withLabel("Person") .withBackendName(DEFAULT_BACKEND_NAME) .withPrimaryKeyField("id") @@ -160,7 +166,7 @@ public class TestUtils public static QTableMetaData definePersonFileTable() { return (new QTableMetaData() - .withName("personFile") + .withName(TABLE_NAME_PERSON_FILE) .withLabel("Person File") .withBackendName(DEFAULT_BACKEND_NAME) .withPrimaryKeyField("id") @@ -175,7 +181,7 @@ public class TestUtils public static QTableMetaData defineTableIdAndNameOnly() { return new QTableMetaData() - .withName("idAndNameOnly") + .withName(TABLE_NAME_ID_AND_NAME_ONLY) .withLabel("Id and Name Only") .withBackendName(DEFAULT_BACKEND_NAME) .withPrimaryKeyField("id") @@ -192,7 +198,7 @@ public class TestUtils { return new QProcessMetaData() .withName(PROCESS_NAME_GREET_PEOPLE) - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .addStep(new QBackendStepMetaData() .withName("prepare") .withCode(new QCodeReference() @@ -200,14 +206,14 @@ public class TestUtils .withCodeType(QCodeType.JAVA) .withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context? .withInputData(new QFunctionInputMetaData() - .withRecordListMetaData(new QRecordListMetaData().withTableName("person")) + .withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON)) .withFieldList(List.of( new QFieldMetaData("greetingPrefix", QFieldType.STRING), new QFieldMetaData("greetingSuffix", QFieldType.STRING) ))) .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) @@ -223,7 +229,7 @@ public class TestUtils { return new QProcessMetaData() .withName(PROCESS_NAME_GREET_PEOPLE_INTERACTIVE) - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .addStep(new QFrontendStepMetaData() .withName("setup") @@ -238,14 +244,14 @@ public class TestUtils .withCodeType(QCodeType.JAVA) .withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context? .withInputData(new QFunctionInputMetaData() - .withRecordListMetaData(new QRecordListMetaData().withTableName("person")) + .withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON)) .withFieldList(List.of( new QFieldMetaData("greetingPrefix", QFieldType.STRING), new QFieldMetaData("greetingSuffix", QFieldType.STRING) ))) .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) @@ -270,8 +276,8 @@ public class TestUtils private static QProcessMetaData defineProcessAddToPeoplesAge() { return new QProcessMetaData() - .withName("addToPeoplesAge") - .withTableName("person") + .withName(PROCESS_NAME_ADD_TO_PEOPLES_AGE) + .withTableName(TABLE_NAME_PERSON) .addStep(new QBackendStepMetaData() .withName("getAgeStatistics") .withCode(new QCodeReference() @@ -279,10 +285,10 @@ public class TestUtils .withCodeType(QCodeType.JAVA) .withCodeUsage(QCodeUsage.BACKEND_STEP)) .withInputData(new QFunctionInputMetaData() - .withRecordListMetaData(new QRecordListMetaData().withTableName("person"))) + .withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON))) .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .addField(new QFieldMetaData("age", QFieldType.INTEGER))) .withFieldList(List.of( new QFieldMetaData("minAge", QFieldType.INTEGER), @@ -297,7 +303,7 @@ public class TestUtils .withFieldList(List.of(new QFieldMetaData("yearsToAdd", QFieldType.INTEGER)))) .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .addField(new QFieldMetaData("newAge", QFieldType.INTEGER))))); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java index a422338a..abb4afbc 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java @@ -24,8 +24,16 @@ package com.kingsrook.qqq.backend.core.utils; import java.math.BigDecimal; import java.math.MathContext; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.Month; +import java.time.ZoneOffset; +import java.util.Calendar; +import java.util.GregorianCalendar; import com.kingsrook.qqq.backend.core.exceptions.QValueException; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.*; @@ -41,6 +49,7 @@ class ValueUtilsTest @Test void testGetValueAsString() throws QValueException { + //noinspection ConstantConditions assertNull(ValueUtils.getValueAsString(null)); assertEquals("", ValueUtils.getValueAsString("")); assertEquals(" ", ValueUtils.getValueAsString(" ")); @@ -132,7 +141,9 @@ class ValueUtilsTest assertEquals(new BigDecimal("1"), ValueUtils.getValueAsBigDecimal(1.0F)); assertEquals(new BigDecimal("1"), ValueUtils.getValueAsBigDecimal(1.0D)); assertEquals(new BigDecimal("1000000000000"), ValueUtils.getValueAsBigDecimal(1_000_000_000_000L)); + //noinspection ConstantConditions assertEquals(0, new BigDecimal("1.1").compareTo(ValueUtils.getValueAsBigDecimal(1.1F).round(MathContext.DECIMAL32))); + //noinspection ConstantConditions assertEquals(0, new BigDecimal("1.1").compareTo(ValueUtils.getValueAsBigDecimal(1.1D).round(MathContext.DECIMAL64))); assertThrows(QValueException.class, () -> ValueUtils.getValueAsBigDecimal("a")); @@ -140,4 +151,79 @@ class ValueUtilsTest assertThrows(QValueException.class, () -> ValueUtils.getValueAsBigDecimal(new Object())); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("deprecation") + @Test + void testGetValueAsLocalDate() throws QValueException + { + assertNull(ValueUtils.getValueAsLocalDate(null)); + assertNull(ValueUtils.getValueAsLocalDate("")); + assertNull(ValueUtils.getValueAsLocalDate(" ")); + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(LocalDate.of(1980, 5, 31))); + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new java.sql.Date(80, 4, 31))); + //noinspection MagicConstant + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new java.util.Date(80, 4, 31))); + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31))); + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31, 12, 0))); + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31, 4, 0))); + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31, 22, 0))); + //noinspection MagicConstant + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new GregorianCalendar(1980, 4, 31))); + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new GregorianCalendar(1980, Calendar.MAY, 31))); + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, 5, 31, 12, 0))); + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, 5, 31, 4, 0))); + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, 5, 31, 22, 0))); + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, Month.MAY, 31, 12, 0))); + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate("1980-05-31")); + assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate("05/31/1980")); + + assertThrows(QValueException.class, () -> ValueUtils.getValueAsLocalDate("a")); + assertThrows(QValueException.class, () -> ValueUtils.getValueAsLocalDate("a,b")); + assertThat(assertThrows(QValueException.class, () -> ValueUtils.getValueAsLocalDate("1980/05/31")).getMessage()).contains("parse"); + assertThat(assertThrows(QValueException.class, () -> ValueUtils.getValueAsLocalDate(new Object())).getMessage()).contains("Unsupported class"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("deprecation") + @Test + void testGetValueAsInstant() throws QValueException + { + Instant expected = Instant.parse("1980-05-31T12:30:00Z"); + + assertNull(ValueUtils.getValueAsInstant(null)); + assertNull(ValueUtils.getValueAsInstant("")); + assertNull(ValueUtils.getValueAsInstant(" ")); + assertEquals(expected, ValueUtils.getValueAsInstant(expected)); + assertEquals(expected, ValueUtils.getValueAsInstant("1980-05-31T12:30:00Z")); + + //////////////////////////// + // todo - time zone logic // + //////////////////////////// + // //noinspection MagicConstant + // assertEquals(expected, ValueUtils.getValueAsInstant(new java.util.Date(80, 4, 31, 7, 30))); + + // //noinspection MagicConstant + // assertEquals(expected, ValueUtils.getValueAsInstant(new GregorianCalendar(1980, 4, 31))); + // assertEquals(expected, ValueUtils.getValueAsInstant(new GregorianCalendar(1980, Calendar.MAY, 31))); + // // assertEquals(expected, ValueUtils.getValueAsInstant(InstantTime.of(1980, 5, 31, 12, 0))); + // // assertEquals(expected, ValueUtils.getValueAsInstant(InstantTime.of(1980, 5, 31, 4, 0))); + // // assertEquals(expected, ValueUtils.getValueAsInstant(InstantTime.of(1980, 5, 31, 22, 0))); + // // assertEquals(expected, ValueUtils.getValueAsInstant(InstantTime.of(1980, Month.MAY, 31, 12, 0))); + + assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant(new java.sql.Date(80, 4, 31))); + assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("a")); + assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("a,b")); + assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("1980/05/31")); + assertThat(assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant(new Object())).getMessage()).contains("Unsupported class"); + } + + } \ No newline at end of file diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index 9a9f1c1a..feafe6c9 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; @@ -94,7 +95,8 @@ public abstract class AbstractRDBMSAction /******************************************************************************* - ** Handle obvious problems with values - like empty string for integer should be null. + ** Handle obvious problems with values - like empty string for integer should be null, + ** and type conversions that we can do "better" than jdbc... ** *******************************************************************************/ protected Serializable scrubValue(QFieldMetaData field, Serializable value, boolean isInsert) @@ -108,6 +110,18 @@ public abstract class AbstractRDBMSAction } } + ////////////////////////////////////////////////////////////////////////////// + // value utils is good at making values from strings - jdbc, not as much... // + ////////////////////////////////////////////////////////////////////////////// + if(field.getType().equals(QFieldType.DATE) && value instanceof String) + { + value = ValueUtils.getValueAsLocalDate(value); + } + else if(field.getType().equals(QFieldType.DECIMAL) && value instanceof String) + { + value = ValueUtils.getValueAsBigDecimal(value); + } + return (value); } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 5c8361fb..02718222 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -189,8 +189,13 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf } case DATE: { + // todo - queryManager.getLocalDate? return (QueryManager.getDate(resultSet, i)); } + case TIME: + { + return (QueryManager.getLocalTime(resultSet, i)); + } case DATE_TIME: { return (QueryManager.getLocalDateTime(resultSet, i)); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java index fbd634e5..be854f39 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java @@ -35,6 +35,7 @@ import java.sql.Types; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.OffsetDateTime; import java.time.ZoneId; import java.time.ZoneOffset; @@ -47,6 +48,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import org.apache.commons.lang.NotImplementedException; @@ -121,7 +123,10 @@ public class QueryManager statement.execute(); resultSet = statement.getResultSet(); - processor.processResultSet(resultSet); + if(processor != null) + { + processor.processResultSet(resultSet); + } } finally { @@ -743,10 +748,14 @@ public class QueryManager } else if(value instanceof LocalDate ld) { - ZoneOffset offset = OffsetDateTime.now().getOffset(); - long epochMillis = ld.atStartOfDay().toEpochSecond(offset) * MS_PER_SEC; - Timestamp timestamp = new Timestamp(epochMillis); - statement.setTimestamp(index, timestamp); + java.sql.Date date = new java.sql.Date(ld.getYear() - 1900, ld.getMonthValue() - 1, ld.getDayOfMonth()); + statement.setDate(index, date); + return (1); + } + else if(value instanceof LocalTime lt) + { + java.sql.Time time = new java.sql.Time(lt.getHour(), lt.getMinute(), lt.getSecond()); + statement.setTime(index, time); return (1); } else if(value instanceof OffsetDateTime odt) @@ -1199,6 +1208,67 @@ public class QueryManager + /******************************************************************************* + ** + *******************************************************************************/ + public static LocalTime getLocalTime(ResultSet resultSet, int column) throws SQLException + { + String timeString = resultSet.getString(column); + if(resultSet.wasNull()) + { + return (null); + } + return stringToLocalTime(timeString); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static LocalTime getLocalTime(ResultSet resultSet, String column) throws SQLException + { + String timeString = resultSet.getString(column); + if(resultSet.wasNull()) + { + return (null); + } + return stringToLocalTime(timeString); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static LocalTime stringToLocalTime(String timeString) throws SQLException + { + if(!StringUtils.hasContent(timeString)) + { + return (null); + } + + String[] parts = timeString.split(":"); + if(parts.length == 1) + { + return LocalTime.of(Integer.parseInt(parts[0]), 0); + } + if(parts.length == 2) + { + return LocalTime.of(Integer.parseInt(parts[0]), Integer.parseInt(parts[1])); + } + else if(parts.length == 3) + { + return LocalTime.of(Integer.parseInt(parts[0]), Integer.parseInt(parts[1]), Integer.parseInt(parts[2])); + } + else + { + throw (new SQLException("Unable to parse time value [" + timeString + "] to LocalTime")); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java index 94978f2a..e62ac2ae 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java @@ -32,6 +32,7 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.Month; import java.time.OffsetDateTime; import java.util.GregorianCalendar; @@ -59,7 +60,16 @@ class QueryManagerTest void beforeEach() throws SQLException { Connection connection = getConnection(); - QueryManager.executeUpdate(connection, "CREATE TABLE t (i INTEGER, dt DATETIME, c CHAR(1), d DATE)"); + QueryManager.executeUpdate(connection, """ + CREATE TABLE test_table + ( + int_col INTEGER, + datetime_col DATETIME, + char_col CHAR(1), + date_col DATE, + time_col TIME + ) + """); } @@ -71,7 +81,7 @@ class QueryManagerTest void afterEach() throws SQLException { Connection connection = getConnection(); - QueryManager.executeUpdate(connection, "DROP TABLE t"); + QueryManager.executeUpdate(connection, "DROP TABLE test_table"); } @@ -95,7 +105,7 @@ class QueryManagerTest { long ctMillis = System.currentTimeMillis(); Connection connection = getConnection(); - PreparedStatement ps = connection.prepareStatement("UPDATE t SET i = ? WHERE i > 0"); + PreparedStatement ps = connection.prepareStatement("UPDATE test_table SET int_col = ? WHERE int_col > 0"); /////////////////////////////////////////////////////////////////////////////// // these calls - we just want to assert that they don't throw any exceptions // @@ -149,37 +159,37 @@ class QueryManagerTest void testGetValueMethods() throws SQLException { Connection connection = getConnection(); - QueryManager.executeUpdate(connection, "INSERT INTO t (i, dt, c) VALUES (1, now(), 'A')"); - PreparedStatement preparedStatement = connection.prepareStatement("SELECT * from t"); + QueryManager.executeUpdate(connection, "INSERT INTO test_table (int_col, datetime_col, char_col) VALUES (1, now(), 'A')"); + PreparedStatement preparedStatement = connection.prepareStatement("SELECT * from test_table"); preparedStatement.execute(); ResultSet rs = preparedStatement.getResultSet(); rs.next(); - assertEquals(1, QueryManager.getInteger(rs, "i")); + assertEquals(1, QueryManager.getInteger(rs, "int_col")); assertEquals(1, QueryManager.getInteger(rs, 1)); - assertEquals(1L, QueryManager.getLong(rs, "i")); + assertEquals(1L, QueryManager.getLong(rs, "int_col")); assertEquals(1L, QueryManager.getLong(rs, 1)); - assertArrayEquals(new byte[] { 0, 0, 0, 1 }, QueryManager.getByteArray(rs, "i")); + assertArrayEquals(new byte[] { 0, 0, 0, 1 }, QueryManager.getByteArray(rs, "int_col")); assertArrayEquals(new byte[] { 0, 0, 0, 1 }, QueryManager.getByteArray(rs, 1)); - assertEquals(1, QueryManager.getObject(rs, "i")); + assertEquals(1, QueryManager.getObject(rs, "int_col")); assertEquals(1, QueryManager.getObject(rs, 1)); - assertEquals(BigDecimal.ONE, QueryManager.getBigDecimal(rs, "i")); + assertEquals(BigDecimal.ONE, QueryManager.getBigDecimal(rs, "int_col")); assertEquals(BigDecimal.ONE, QueryManager.getBigDecimal(rs, 1)); - assertEquals(true, QueryManager.getBoolean(rs, "i")); + assertEquals(true, QueryManager.getBoolean(rs, "int_col")); assertEquals(true, QueryManager.getBoolean(rs, 1)); - assertNotNull(QueryManager.getDate(rs, "dt")); + assertNotNull(QueryManager.getDate(rs, "datetime_col")); assertNotNull(QueryManager.getDate(rs, 2)); - assertNotNull(QueryManager.getCalendar(rs, "dt")); + assertNotNull(QueryManager.getCalendar(rs, "datetime_col")); assertNotNull(QueryManager.getCalendar(rs, 2)); - assertNotNull(QueryManager.getLocalDate(rs, "dt")); + assertNotNull(QueryManager.getLocalDate(rs, "datetime_col")); assertNotNull(QueryManager.getLocalDate(rs, 2)); - assertNotNull(QueryManager.getLocalDateTime(rs, "dt")); + assertNotNull(QueryManager.getLocalDateTime(rs, "datetime_col")); assertNotNull(QueryManager.getLocalDateTime(rs, 2)); - assertNotNull(QueryManager.getOffsetDateTime(rs, "dt")); + assertNotNull(QueryManager.getOffsetDateTime(rs, "datetime_col")); assertNotNull(QueryManager.getOffsetDateTime(rs, 2)); - assertNotNull(QueryManager.getTimestamp(rs, "dt")); + assertNotNull(QueryManager.getTimestamp(rs, "datetime_col")); assertNotNull(QueryManager.getTimestamp(rs, 2)); - assertEquals("A", QueryManager.getObject(rs, "c")); + assertEquals("A", QueryManager.getObject(rs, "char_col")); assertEquals("A", QueryManager.getObject(rs, 3)); } @@ -192,37 +202,37 @@ class QueryManagerTest void testGetValueMethodsReturningNull() throws SQLException { Connection connection = getConnection(); - QueryManager.executeUpdate(connection, "INSERT INTO t (i, dt, c) VALUES (null, null, null)"); - PreparedStatement preparedStatement = connection.prepareStatement("SELECT * from t"); + QueryManager.executeUpdate(connection, "INSERT INTO test_table (int_col, datetime_col, char_col) VALUES (null, null, null)"); + PreparedStatement preparedStatement = connection.prepareStatement("SELECT * from test_table"); preparedStatement.execute(); ResultSet rs = preparedStatement.getResultSet(); rs.next(); - assertNull(QueryManager.getInteger(rs, "i")); + assertNull(QueryManager.getInteger(rs, "int_col")); assertNull(QueryManager.getInteger(rs, 1)); - assertNull(QueryManager.getLong(rs, "i")); + assertNull(QueryManager.getLong(rs, "int_col")); assertNull(QueryManager.getLong(rs, 1)); - assertNull(QueryManager.getByteArray(rs, "i")); + assertNull(QueryManager.getByteArray(rs, "int_col")); assertNull(QueryManager.getByteArray(rs, 1)); - assertNull(QueryManager.getObject(rs, "i")); + assertNull(QueryManager.getObject(rs, "int_col")); assertNull(QueryManager.getObject(rs, 1)); - assertNull(QueryManager.getBigDecimal(rs, "i")); + assertNull(QueryManager.getBigDecimal(rs, "int_col")); assertNull(QueryManager.getBigDecimal(rs, 1)); - assertNull(QueryManager.getBoolean(rs, "i")); + assertNull(QueryManager.getBoolean(rs, "int_col")); assertNull(QueryManager.getBoolean(rs, 1)); - assertNull(QueryManager.getDate(rs, "dt")); + assertNull(QueryManager.getDate(rs, "datetime_col")); assertNull(QueryManager.getDate(rs, 2)); - assertNull(QueryManager.getCalendar(rs, "dt")); + assertNull(QueryManager.getCalendar(rs, "datetime_col")); assertNull(QueryManager.getCalendar(rs, 2)); - assertNull(QueryManager.getLocalDate(rs, "dt")); + assertNull(QueryManager.getLocalDate(rs, "datetime_col")); assertNull(QueryManager.getLocalDate(rs, 2)); - assertNull(QueryManager.getLocalDateTime(rs, "dt")); + assertNull(QueryManager.getLocalDateTime(rs, "datetime_col")); assertNull(QueryManager.getLocalDateTime(rs, 2)); - assertNull(QueryManager.getOffsetDateTime(rs, "dt")); + assertNull(QueryManager.getOffsetDateTime(rs, "datetime_col")); assertNull(QueryManager.getOffsetDateTime(rs, 2)); - assertNull(QueryManager.getTimestamp(rs, "dt")); + assertNull(QueryManager.getTimestamp(rs, "datetime_col")); assertNull(QueryManager.getTimestamp(rs, 2)); - assertNull(QueryManager.getObject(rs, "c")); + assertNull(QueryManager.getObject(rs, "char_col")); assertNull(QueryManager.getObject(rs, 3)); } @@ -236,9 +246,9 @@ class QueryManagerTest void testLocalDate() throws SQLException { Connection connection = getConnection(); - QueryManager.executeUpdate(connection, "INSERT INTO t (d) VALUES (?)", LocalDate.of(2013, Month.OCTOBER, 1)); + QueryManager.executeUpdate(connection, "INSERT INTO test_table (date_col) VALUES (?)", LocalDate.of(2013, Month.OCTOBER, 1)); - PreparedStatement preparedStatement = connection.prepareStatement("SELECT d from t"); + PreparedStatement preparedStatement = connection.prepareStatement("SELECT date_col from test_table"); preparedStatement.execute(); ResultSet rs = preparedStatement.getResultSet(); rs.next(); @@ -268,4 +278,55 @@ class QueryManagerTest assertEquals(0, offsetDateTime.getMinute(), "Minute value"); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLocalTime() throws SQLException + { + Connection connection = getConnection(); + + //////////////////////////////////// + // insert one just hour & minutes // + //////////////////////////////////// + QueryManager.executeUpdate(connection, "INSERT INTO test_table (int_col, time_col) VALUES (?, ?)", 1, LocalTime.of(10, 42)); + + PreparedStatement preparedStatement = connection.prepareStatement("SELECT time_col from test_table where int_col=1"); + preparedStatement.execute(); + ResultSet rs = preparedStatement.getResultSet(); + rs.next(); + + LocalTime localTime = QueryManager.getLocalTime(rs, 1); + assertEquals(10, localTime.getHour(), "Hour value"); + assertEquals(42, localTime.getMinute(), "Minute value"); + assertEquals(0, localTime.getSecond(), "Second value"); + + localTime = QueryManager.getLocalTime(rs, "time_col"); + assertEquals(10, localTime.getHour(), "Hour value"); + assertEquals(42, localTime.getMinute(), "Minute value"); + assertEquals(0, localTime.getSecond(), "Second value"); + + ///////////////////////////////// + // now insert one with seconds // + ///////////////////////////////// + QueryManager.executeUpdate(connection, "INSERT INTO test_table (int_col, time_col) VALUES (?, ?)", 2, LocalTime.of(10, 42, 59)); + + preparedStatement = connection.prepareStatement("SELECT time_col from test_table where int_col=2"); + preparedStatement.execute(); + rs = preparedStatement.getResultSet(); + rs.next(); + + localTime = QueryManager.getLocalTime(rs, 1); + assertEquals(10, localTime.getHour(), "Hour value"); + assertEquals(42, localTime.getMinute(), "Minute value"); + assertEquals(59, localTime.getSecond(), "Second value"); + + localTime = QueryManager.getLocalTime(rs, "time_col"); + assertEquals(10, localTime.getHour(), "Hour value"); + assertEquals(42, localTime.getMinute(), "Minute value"); + assertEquals(59, localTime.getSecond(), "Second value"); + } + } \ No newline at end of file diff --git a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java index 505f1340..b1e18017 100644 --- a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java +++ b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.frontend.picocli; import java.math.BigDecimal; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -407,7 +408,8 @@ public class QCommandBuilder case INTEGER -> Integer.class; case DECIMAL -> BigDecimal.class; case DATE -> LocalDate.class; - // case TIME -> LocalTime.class; + case TIME -> LocalTime.class; + case BOOLEAN -> Boolean.class; case DATE_TIME -> LocalDateTime.class; case BLOB -> byte[].class; }; From 22b322fe83777312b5eea5d95d1fd6fe853f1a72 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 2 Aug 2022 09:01:47 -0500 Subject: [PATCH 02/18] Try to fix deployed jars references to parent by specifying -Drevision in deploy command --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a42afca5..b629f771 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,7 +69,7 @@ commands: - run: name: Run Maven Jar Deploy command: | - mvn -s .circleci/mvn-settings.xml jar:jar deploy:deploy + mvn -s -Drevision=0.3.0-SNAPSHOT .circleci/mvn-settings.xml jar:jar deploy:deploy - save_cache: paths: - ~/.m2 From 010ee595cea401c34824448341c1daa3bd29c0ad Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 2 Aug 2022 09:10:20 -0500 Subject: [PATCH 03/18] Revert previous (incorrectly addded -Drevision); Add flatten-maven-plugin --- .circleci/config.yml | 2 +- pom.xml | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b629f771..a42afca5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,7 +69,7 @@ commands: - run: name: Run Maven Jar Deploy command: | - mvn -s -Drevision=0.3.0-SNAPSHOT .circleci/mvn-settings.xml jar:jar deploy:deploy + mvn -s .circleci/mvn-settings.xml jar:jar deploy:deploy - save_cache: paths: - ~/.m2 diff --git a/pom.xml b/pom.xml index 0e78e1d0..6664339e 100644 --- a/pom.xml +++ b/pom.xml @@ -137,6 +137,31 @@ + + org.codehaus.mojo + flatten-maven-plugin + 1.1.0 + + true + resolveCiFriendliesOnly + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + com.amashchenko.maven.plugin gitflow-maven-plugin From b5b4ded8aa578d390606d82c1ff2c6cc429eeff2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 2 Aug 2022 09:14:54 -0500 Subject: [PATCH 04/18] Add call to flatten:flatten --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a42afca5..3e7638e3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,7 +69,7 @@ commands: - run: name: Run Maven Jar Deploy command: | - mvn -s .circleci/mvn-settings.xml jar:jar deploy:deploy + mvn -s .circleci/mvn-settings.xml flatten:flatten jar:jar deploy:deploy - save_cache: paths: - ~/.m2 From a2e267fe40e38a934b2ac18c052f9c635b018a61 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 4 Aug 2022 11:34:19 -0500 Subject: [PATCH 05/18] Changes pushed to qqq-backend-core (solo-repo) in 0.2 support --- .../backend/core/actions/ActionHelper.java | 5 +- .../core/actions/QBackendTransaction.java | 82 ++++++ .../actions/interfaces/InsertInterface.java | 10 + .../core/actions/reporting/RecordPipe.java | 21 +- .../core/actions/tables/InsertAction.java | 30 ++- .../core/adapters/CsvToQRecordAdapter.java | 83 +++++- .../actions/tables/insert/InsertInput.java | 35 +++ .../tables/query/QueryOutputRecordPipe.java | 33 +-- .../Auth0AuthenticationModule.java | 91 +++++-- .../FullyAnonymousAuthenticationModule.java | 2 +- .../MockAuthenticationModule.java | 2 +- .../QAuthenticationModuleInterface.java | 2 +- .../etl/basic/BasicETLExtractFunction.java | 32 ++- .../etl/basic/BasicETLLoadFunction.java | 17 +- .../etl/streamed/StreamedETLBackendStep.java | 240 ++++++++++++++++++ .../etl/streamed/StreamedETLProcess.java | 75 ++++++ .../actions/reporting/ReportActionTest.java | 2 +- .../Auth0AuthenticationModuleTest.java | 89 +++++-- ...ullyAnonymousAuthenticationModuleTest.java | 4 +- .../etl/streamed/StreamedETLProcessTest.java | 89 +++++++ 20 files changed, 840 insertions(+), 104 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcessTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java index d5ab7318..11ef0eeb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java @@ -34,6 +34,9 @@ import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModu *******************************************************************************/ public class ActionHelper { + private int f; + + /******************************************************************************* ** @@ -42,7 +45,7 @@ public class ActionHelper { QAuthenticationModuleDispatcher qAuthenticationModuleDispatcher = new QAuthenticationModuleDispatcher(); QAuthenticationModuleInterface authenticationModule = qAuthenticationModuleDispatcher.getQModule(request.getAuthenticationMetaData()); - if(!authenticationModule.isSessionValid(request.getSession())) + if(!authenticationModule.isSessionValid(request.getInstance(), request.getSession())) { throw new QAuthenticationException("Invalid session in request"); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java new file mode 100644 index 00000000..4220ec90 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java @@ -0,0 +1,82 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions; + + +import java.io.IOException; +import com.kingsrook.qqq.backend.core.exceptions.QException; + + +/******************************************************************************* + ** Container wherein backend modules can track data and/or objects that are + ** part of a transaction. + ** + ** Most obvious use-case would be a JDBC Connection. See subclass in rdbms module. + *******************************************************************************/ +public class QBackendTransaction +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public void commit() throws QException + { + //////////////////////// + // noop in base class // + //////////////////////// + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void rollback() throws QException + { + //////////////////////// + // noop in base class // + //////////////////////// + } + + + + /******************************************************************************* + * Closes this stream and releases any system resources associated + * with it. If the stream is already closed then invoking this + * method has no effect. + * + *

As noted in {@link AutoCloseable#close()}, cases where the + * close may fail require careful attention. It is strongly advised + * to relinquish the underlying resources and to internally + * mark the {@code Closeable} as closed, prior to throwing + * the {@code IOException}. + * + * @throws IOException + * if an I/O error occurs + *******************************************************************************/ + public void close() + { + //////////////////////// + // noop in base class // + //////////////////////// + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java index 34e79045..dc90de66 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.actions.interfaces; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; @@ -37,4 +38,13 @@ public interface InsertInterface ** *******************************************************************************/ InsertOutput execute(InsertInput insertInput) throws QException; + + /******************************************************************************* + ** + *******************************************************************************/ + default QBackendTransaction openTransaction(InsertInput insertInput) throws QException + { + return (new QBackendTransaction()); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java index cb8e3b6d..3703a841 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java @@ -25,7 +25,11 @@ package com.kingsrook.qqq.backend.core.actions.reporting; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -34,16 +38,25 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; *******************************************************************************/ public class RecordPipe { - private ArrayBlockingQueue queue = new ArrayBlockingQueue<>(10_000); + private static final Logger LOG = LogManager.getLogger(RecordPipe.class); + + private ArrayBlockingQueue queue = new ArrayBlockingQueue<>(1_000); /******************************************************************************* ** Add a record to the pipe ** Returns true iff the record fit in the pipe; false if the pipe is currently full. *******************************************************************************/ - public boolean addRecord(QRecord record) + public void addRecord(QRecord record) { - return (queue.offer(record)); + boolean offerResult = queue.offer(record); + + while(!offerResult) + { + LOG.debug("Record pipe.add failed (due to full pipe). Blocking."); + SleepUtils.sleep(100, TimeUnit.MILLISECONDS); + offerResult = queue.offer(record); + } } @@ -53,7 +66,7 @@ public class RecordPipe *******************************************************************************/ public void addRecords(List records) { - queue.addAll(records); + records.forEach(this::addRecord); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index d4f6cd92..54ebcfb2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.tables; import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; @@ -47,14 +48,35 @@ public class InsertAction *******************************************************************************/ public InsertOutput execute(InsertInput insertInput) throws QException { - ActionHelper.validateSession(insertInput); - - QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); - QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(insertInput.getBackend()); + QBackendModuleInterface qModule = getBackendModuleInterface(insertInput); // todo pre-customization - just get to modify the request? InsertOutput insertOutput = qModule.getInsertInterface().execute(insertInput); // todo post-customization - can do whatever w/ the result if you want return insertOutput; } + + + /******************************************************************************* + ** + *******************************************************************************/ + private QBackendModuleInterface getBackendModuleInterface(InsertInput insertInput) throws QException + { + ActionHelper.validateSession(insertInput); + QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); + QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(insertInput.getBackend()); + return (qModule); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QBackendTransaction openTransaction(InsertInput insertInput) throws QException + { + QBackendModuleInterface qModule = getBackendModuleInterface(insertInput); + return (qModule.getInsertInterface().openTransaction(insertInput)); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java index 38b7f58a..3d5493a3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java @@ -28,6 +28,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -41,9 +43,44 @@ import org.apache.commons.csv.CSVRecord; /******************************************************************************* ** Adapter class to convert a CSV string into a list of QRecords. ** + ** Based on which method is called, can either take a pipe, and stream records + ** into it - or return a list of all records from the file. Either way, at this + ** time, the full CSV string is read & parsed - a future optimization might read + ** the CSV content from a stream as well. *******************************************************************************/ public class CsvToQRecordAdapter { + private RecordPipe recordPipe = null; + private List recordList = null; + + + + /******************************************************************************* + ** stream records from a CSV String into a RecordPipe, for a given table, optionally + ** using a given mapping. + ** + *******************************************************************************/ + public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping mapping, Consumer recordCustomizer) + { + this.recordPipe = recordPipe; + doBuildRecordsFromCsv(csv, table, mapping, recordCustomizer); + } + + + + /******************************************************************************* + ** convert a CSV String into a List of QRecords, for a given table, optionally + ** using a given mapping. + ** + *******************************************************************************/ + public List buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping mapping) + { + this.recordList = new ArrayList<>(); + doBuildRecordsFromCsv(csv, table, mapping, null); + return (recordList); + } + + /******************************************************************************* ** convert a CSV String into a List of QRecords, for a given table, optionally @@ -51,14 +88,13 @@ public class CsvToQRecordAdapter ** ** todo - meta-data validation, type handling *******************************************************************************/ - public List buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping mapping) + public void doBuildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping mapping, Consumer recordCustomizer) { if(!StringUtils.hasContent(csv)) { throw (new IllegalArgumentException("Empty csv value was provided.")); } - List rs = new ArrayList<>(); try { /////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -82,7 +118,7 @@ public class CsvToQRecordAdapter // put values from the CSV record into a map of header -> value // ////////////////////////////////////////////////////////////////// Map csvValues = new HashMap<>(); - for(int i=0; i recordCustomizer, QRecord qRecord) + { + if(recordCustomizer != null) + { + recordCustomizer.accept(qRecord); + } } @@ -165,7 +216,7 @@ public class CsvToQRecordAdapter for(String header : headers) { - String headerToUse = header; + String headerToUse = header; String headerWithoutSuffix = header.replaceFirst(" \\d+$", ""); if(countsByHeader.containsKey(headerWithoutSuffix)) @@ -183,4 +234,22 @@ public class CsvToQRecordAdapter return (rs); } + + + /******************************************************************************* + ** Add a record - either to the pipe, or list, whichever we're building. + *******************************************************************************/ + private void addRecord(QRecord record) + { + if(recordPipe != null) + { + recordPipe.addRecord(record); + } + + if(recordList != null) + { + recordList.add(record); + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java index d2ec4f15..24320d98 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.insert; import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -34,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; *******************************************************************************/ public class InsertInput extends AbstractTableActionInput { + private QBackendTransaction transaction; private List records; @@ -57,6 +59,39 @@ public class InsertInput extends AbstractTableActionInput + /******************************************************************************* + ** Getter for transaction + ** + *******************************************************************************/ + public QBackendTransaction getTransaction() + { + return transaction; + } + + + + /******************************************************************************* + ** Setter for transaction + ** + *******************************************************************************/ + public void setTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + } + + + /******************************************************************************* + ** Fluent setter for transaction + ** + *******************************************************************************/ + public InsertInput withTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + return (this); + } + + + /******************************************************************************* ** Getter for records ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java index 9e65ade9..6f8ce386 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java @@ -23,10 +23,8 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.util.List; -import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.utils.SleepUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -58,35 +56,7 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface @Override public void addRecord(QRecord record) { - if(!recordPipe.addRecord(record)) - { - do - { - LOG.debug("Record pipe.add failed (due to full pipe). Blocking."); - SleepUtils.sleep(10, TimeUnit.MILLISECONDS); - } - while(!recordPipe.addRecord(record)); - LOG.debug("Done blocking."); - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void blockIfPipeIsTooFull() - { - if(recordPipe.countAvailableRecords() >= 100_000) - { - LOG.info("Record pipe is kinda full. Blocking for a bit"); - do - { - SleepUtils.sleep(10, TimeUnit.MILLISECONDS); - } - while(recordPipe.countAvailableRecords() >= 10_000); - LOG.info("Done blocking."); - } + recordPipe.addRecord(record); } @@ -98,7 +68,6 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface public void addRecords(List records) { recordPipe.addRecords(records); - blockIfPipeIsTooFull(); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModule.java index 79bc0d73..435a13ce 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModule.java @@ -57,20 +57,25 @@ import org.json.JSONObject; *******************************************************************************/ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface { - private static final Logger logger = LogManager.getLogger(Auth0AuthenticationModule.class); + private static final Logger LOG = LogManager.getLogger(Auth0AuthenticationModule.class); - private static final int ID_TOKEN_VALIDATION_INTERVAL_SECONDS = 300; + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // 30 minutes - ideally this would be lower, but right now we've been dealing with re-validation issues... // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + public static final int ID_TOKEN_VALIDATION_INTERVAL_SECONDS = 1800; public static final String AUTH0_ID_TOKEN_KEY = "sessionId"; public static final String TOKEN_NOT_PROVIDED_ERROR = "Id Token was not provided"; - public static final String COULD_NOT_DECODE_ERROR = "Unable to decode id token"; - public static final String EXPIRED_TOKEN_ERROR = "Token has expired"; - public static final String INVALID_TOKEN_ERROR = "An invalid token was provided"; + public static final String COULD_NOT_DECODE_ERROR = "Unable to decode id token"; + public static final String EXPIRED_TOKEN_ERROR = "Token has expired"; + public static final String INVALID_TOKEN_ERROR = "An invalid token was provided"; private Instant now; + + /******************************************************************************* ** *******************************************************************************/ @@ -83,7 +88,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface String idToken = context.get(AUTH0_ID_TOKEN_KEY); if(idToken == null) { - logger.warn(TOKEN_NOT_PROVIDED_ERROR); + LOG.warn(TOKEN_NOT_PROVIDED_ERROR); throw (new QAuthenticationException(TOKEN_NOT_PROVIDED_ERROR)); } @@ -97,7 +102,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // then call method to check more session validity // ///////////////////////////////////////////////////// QSession qSession = buildQSessionFromToken(idToken); - if(isSessionValid(qSession)) + if(isSessionValid(qInstance, qSession)) { return (qSession); } @@ -112,7 +117,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // put now into state so we dont check until next interval passes // /////////////////////////////////////////////////////////////////// StateProviderInterface spi = getStateProvider(); - Auth0StateKey key = new Auth0StateKey(qSession.getIdReference()); + Auth0StateKey key = new Auth0StateKey(qSession.getIdReference()); spi.put(key, Instant.now()); return (qSession); @@ -122,12 +127,12 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface //////////////////////////////// // could not decode the token // //////////////////////////////// - logger.warn(COULD_NOT_DECODE_ERROR, jde); + LOG.warn(COULD_NOT_DECODE_ERROR, jde); throw (new QAuthenticationException(COULD_NOT_DECODE_ERROR)); } catch(TokenExpiredException tee) { - logger.info(EXPIRED_TOKEN_ERROR, tee); + LOG.info(EXPIRED_TOKEN_ERROR, tee); throw (new QAuthenticationException(EXPIRED_TOKEN_ERROR)); } catch(JWTVerificationException | JwkException jve) @@ -135,7 +140,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface /////////////////////////////////////////// // token had invalid signature or claims // /////////////////////////////////////////// - logger.warn(INVALID_TOKEN_ERROR, jve); + LOG.warn(INVALID_TOKEN_ERROR, jve); throw (new QAuthenticationException(INVALID_TOKEN_ERROR)); } catch(Exception e) @@ -144,7 +149,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // ¯\_(ツ)_/¯ // //////////////// String message = "An unknown error occurred"; - logger.error(message, e); + LOG.error(message, e); throw (new QAuthenticationException(message)); } } @@ -155,16 +160,16 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface ** *******************************************************************************/ @Override - public boolean isSessionValid(QSession session) + public boolean isSessionValid(QInstance instance, QSession session) { if(session == null) { return (false); } - StateProviderInterface spi = getStateProvider(); - Auth0StateKey key = new Auth0StateKey(session.getIdReference()); - Optional lastTimeCheckedOptional = spi.get(Instant.class, key); + StateProviderInterface spi = getStateProvider(); + Auth0StateKey key = new Auth0StateKey(session.getIdReference()); + Optional lastTimeCheckedOptional = spi.get(Instant.class, key); if(lastTimeCheckedOptional.isPresent()) { Instant lastTimeChecked = lastTimeCheckedOptional.get(); @@ -174,7 +179,28 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // - so this is basically saying, if the time between the last time we checked the token and // // right now is more than ID_TOKEN_VALIDATION_INTERVAL_SECTIONS, then session needs revalidated // /////////////////////////////////////////////////////////////////////////////////////////////////// - return (Duration.between(lastTimeChecked, Instant.now()).compareTo(Duration.ofSeconds(ID_TOKEN_VALIDATION_INTERVAL_SECONDS)) < 0); + if(Duration.between(lastTimeChecked, Instant.now()).compareTo(Duration.ofSeconds(ID_TOKEN_VALIDATION_INTERVAL_SECONDS)) < 0) + { + return (true); + } + + try + { + LOG.debug("Re-validating token due to validation interval being passed: " + session.getIdReference()); + revalidateToken(instance, session.getIdReference()); + + ////////////////////////////////////////////////////////////////// + // update the timestamp in state provider, to avoid re-checking // + ////////////////////////////////////////////////////////////////// + spi.put(key, Instant.now()); + + return (true); + } + catch(Exception e) + { + LOG.warn(INVALID_TOKEN_ERROR, e); + return (false); + } } return (false); @@ -190,10 +216,10 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface { Auth0AuthenticationMetaData metaData = (Auth0AuthenticationMetaData) qInstance.getAuthentication(); - DecodedJWT jwt = JWT.decode(idToken); - JwkProvider provider = new UrlJwkProvider(metaData.getBaseUrl()); - Jwk jwk = provider.get(jwt.getKeyId()); - Algorithm algorithm = Algorithm.RSA256((RSAPublicKey) jwk.getPublicKey(), null); + DecodedJWT jwt = JWT.decode(idToken); + JwkProvider provider = new UrlJwkProvider(metaData.getBaseUrl()); + Jwk jwk = provider.get(jwt.getKeyId()); + Algorithm algorithm = Algorithm.RSA256((RSAPublicKey) jwk.getPublicKey(), null); JWTVerifier verifier = JWT.require(algorithm) .withIssuer(metaData.getBaseUrl()) .build(); @@ -217,20 +243,31 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface //////////////////////////////////// // decode and extract the payload // //////////////////////////////////// - DecodedJWT jwt = JWT.decode(idToken); - Base64.Decoder decoder = Base64.getUrlDecoder(); - String payloadString = new String(decoder.decode(jwt.getPayload())); - JSONObject payload = new JSONObject(payloadString); + DecodedJWT jwt = JWT.decode(idToken); + Base64.Decoder decoder = Base64.getUrlDecoder(); + String payloadString = new String(decoder.decode(jwt.getPayload())); + JSONObject payload = new JSONObject(payloadString); QUser qUser = new QUser(); - qUser.setFullName(payload.getString("name")); + if(payload.has("name")) + { + qUser.setFullName(payload.getString("name")); + } + else + { + qUser.setFullName("Unknown"); + } + if(payload.has("email")) { qUser.setIdReference(payload.getString("email")); } else { - qUser.setIdReference(payload.getString("nickname")); + if(payload.has("sub")) + { + qUser.setIdReference(payload.getString("sub")); + } } QSession qSession = new QSession(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModule.java index 442941b5..8a0e7078 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModule.java @@ -66,7 +66,7 @@ public class FullyAnonymousAuthenticationModule implements QAuthenticationModule ** *******************************************************************************/ @Override - public boolean isSessionValid(QSession session) + public boolean isSessionValid(QInstance instance, QSession session) { return session != null; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/MockAuthenticationModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/MockAuthenticationModule.java index 21b25c2a..f490d0a4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/MockAuthenticationModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/MockAuthenticationModule.java @@ -63,7 +63,7 @@ public class MockAuthenticationModule implements QAuthenticationModuleInterface ** *******************************************************************************/ @Override - public boolean isSessionValid(QSession session) + public boolean isSessionValid(QInstance instance, QSession session) { if(session == null) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java index ec5db589..75c1535f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/QAuthenticationModuleInterface.java @@ -43,5 +43,5 @@ public interface QAuthenticationModuleInterface /******************************************************************************* ** *******************************************************************************/ - boolean isSessionValid(QSession session); + boolean isSessionValid(QInstance instance, QSession session); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java index 20c27808..9b0d0687 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -40,6 +41,8 @@ public class BasicETLExtractFunction implements BackendStep { private static final Logger LOG = LogManager.getLogger(BasicETLExtractFunction.class); + private RecordPipe recordPipe = null; + /******************************************************************************* @@ -64,10 +67,35 @@ public class BasicETLExtractFunction implements BackendStep // queryRequest.setFilter(JsonUtils.toObject(filter, QQueryFilter.class)); // } + ////////////////////////////////////////////////////////////////////// + // if the caller gave us a record pipe, pass it to the query action // + ////////////////////////////////////////////////////////////////////// + if (recordPipe != null) + { + queryInput.setRecordPipe(recordPipe); + } + QueryAction queryAction = new QueryAction(); QueryOutput queryOutput = queryAction.execute(queryInput); - runBackendStepOutput.setRecords(queryOutput.getRecords()); - LOG.info("Query on table " + tableName + " produced " + queryOutput.getRecords().size() + " records."); + if (recordPipe == null) + { + //////////////////////////////////////////////////////////////////////////// + // only return the records (and log about them) if there's no record pipe // + //////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.setRecords(queryOutput.getRecords()); + LOG.info("Query on table " + tableName + " produced " + queryOutput.getRecords().size() + " records."); + } + } + + + + /******************************************************************************* + ** Setter for recordPipe + ** + *******************************************************************************/ + public void setRecordPipe(RecordPipe recordPipe) + { + this.recordPipe = recordPipe; } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java index 997e377b..0ce1c572 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -44,6 +45,8 @@ public class BasicETLLoadFunction implements BackendStep { private static final Logger LOG = LogManager.getLogger(BasicETLLoadFunction.class); + private QBackendTransaction transaction; + /******************************************************************************* @@ -86,10 +89,11 @@ public class BasicETLLoadFunction implements BackendStep insertInput.setSession(runBackendStepInput.getSession()); insertInput.setTableName(table); insertInput.setRecords(page); + insertInput.setTransaction(transaction); InsertAction insertAction = new InsertAction(); InsertOutput insertOutput = insertAction.execute(insertInput); - outputRecords.addAll(insertOutput.getRecords()); + // todo - this is to avoid garbage leak in state provider... outputRecords.addAll(insertOutput.getRecords()); recordsInserted += insertOutput.getRecords().size(); } @@ -97,4 +101,15 @@ public class BasicETLLoadFunction implements BackendStep runBackendStepOutput.addValue(BasicETLProcess.FIELD_RECORD_COUNT, recordsInserted); } + + + /******************************************************************************* + ** Setter for transaction + ** + *******************************************************************************/ + public void setTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java new file mode 100644 index 00000000..bb4337dc --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java @@ -0,0 +1,240 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed; + + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager; +import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState; +import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLExtractFunction; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLLoadFunction; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLTransformFunction; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Backend step to do a streamed ETL + *******************************************************************************/ +public class StreamedETLBackendStep implements BackendStep +{ + private static final Logger LOG = LogManager.getLogger(StreamedETLBackendStep.class); + + private static final int TIMEOUT_AFTER_NO_RECORDS_MS = 10 * 60 * 1000; + + private static final int MAX_SLEEP_MS = 1000; + private static final int INIT_SLEEP_MS = 10; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + QBackendTransaction transaction = openTransaction(runBackendStepInput); + + try + { + RecordPipe recordPipe = new RecordPipe(); + BasicETLExtractFunction basicETLExtractFunction = new BasicETLExtractFunction(); + basicETLExtractFunction.setRecordPipe(recordPipe); + + ////////////////////////////////////////// + // run the query action as an async job // + ////////////////////////////////////////// + AsyncJobManager asyncJobManager = new AsyncJobManager(); + String queryJobUUID = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> + { + basicETLExtractFunction.run(runBackendStepInput, runBackendStepOutput); + return (runBackendStepOutput); + }); + LOG.info("Started query job [" + queryJobUUID + "] for report"); + + AsyncJobState queryJobState = AsyncJobState.RUNNING; + AsyncJobStatus asyncJobStatus = null; + + long recordCount = 0; + int nextSleepMillis = INIT_SLEEP_MS; + long lastReceivedRecordsAt = System.currentTimeMillis(); + long jobStartTime = System.currentTimeMillis(); + + while(queryJobState.equals(AsyncJobState.RUNNING)) + { + if(recordPipe.countAvailableRecords() == 0) + { + /////////////////////////////////////////////////////////// + // if the pipe is empty, sleep to let the producer work. // + // todo - smarter sleep? like get notified vs. sleep? // + /////////////////////////////////////////////////////////// + LOG.info("No records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work"); + SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS); + nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS); + + long timeSinceLastReceivedRecord = System.currentTimeMillis() - lastReceivedRecordsAt; + if(timeSinceLastReceivedRecord > TIMEOUT_AFTER_NO_RECORDS_MS) + { + throw (new QException("Query action appears to have stopped producing records (last record received " + timeSinceLastReceivedRecord + " ms ago).")); + } + } + else + { + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the pipe has records, consume them. reset the sleep timer so if we sleep again it'll be short. // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + lastReceivedRecordsAt = System.currentTimeMillis(); + nextSleepMillis = INIT_SLEEP_MS; + + recordCount += consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction); + + LOG.info(String.format("Processed %,d records so far", recordCount)); + } + + //////////////////////////////////// + // refresh the query job's status // + //////////////////////////////////// + Optional optionalAsyncJobStatus = asyncJobManager.getJobStatus(queryJobUUID); + if(optionalAsyncJobStatus.isEmpty()) + { + ///////////////////////////////////////////////// + // todo - ... maybe some version of try-again? // + ///////////////////////////////////////////////// + throw (new QException("Could not get status of report query job [" + queryJobUUID + "]")); + } + asyncJobStatus = optionalAsyncJobStatus.get(); + queryJobState = asyncJobStatus.getState(); + } + + LOG.info("Query job [" + queryJobUUID + "] for ETL completed with status: " + asyncJobStatus); + + ////////////////////////////////////////////////////// + // send the final records to transform & load steps // + ////////////////////////////////////////////////////// + recordCount += consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction); + + ///////////////////// + // commit the work // + ///////////////////// + transaction.commit(); + + long reportEndTime = System.currentTimeMillis(); + LOG.info(String.format("Processed %,d records", recordCount) + + String.format(" at end of ETL job in %,d ms (%.2f records/second).", (reportEndTime - jobStartTime), 1000d * (recordCount / (.001d + (reportEndTime - jobStartTime))))); + + runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount); + } + catch(Exception e) + { + //////////////////////////////////////////////////////////////////////////////// + // rollback the work, then re-throw the error for up-stream to catch & report // + //////////////////////////////////////////////////////////////////////////////// + transaction.rollback(); + throw (e); + } + finally + { + //////////////////////////////////////////////////////////// + // always close our transactions (e.g., jdbc connections) // + //////////////////////////////////////////////////////////// + transaction.close(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QBackendTransaction openTransaction(RunBackendStepInput runBackendStepInput) throws QException + { + InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance()); + + insertInput.setSession(runBackendStepInput.getSession()); + insertInput.setTableName(runBackendStepInput.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE)); + + return new InsertAction().openTransaction(insertInput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private int consumeRecordsFromPipe(RecordPipe recordPipe, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, QBackendTransaction transaction) throws QException + { + List qRecords = recordPipe.consumeAvailableRecords(); + + preTransform(qRecords, runBackendStepInput, runBackendStepOutput); + + runBackendStepInput.setRecords(qRecords); + new BasicETLTransformFunction().run(runBackendStepInput, runBackendStepOutput); + + postTransform(qRecords, runBackendStepInput, runBackendStepOutput); + + runBackendStepInput.setRecords(runBackendStepOutput.getRecords()); + BasicETLLoadFunction basicETLLoadFunction = new BasicETLLoadFunction(); + basicETLLoadFunction.setTransaction(transaction); + basicETLLoadFunction.run(runBackendStepInput, runBackendStepOutput); + + return (qRecords.size()); + } + + + + /******************************************************************************* + ** Customization point for subclasses of this step. + *******************************************************************************/ + protected void preTransform(List qRecords, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + //////////////////////// + // noop in base class // + //////////////////////// + } + + + + /******************************************************************************* + ** Customization point for subclasses of this step. + *******************************************************************************/ + protected void postTransform(List qRecords, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + //////////////////////// + // noop in base class // + //////////////////////// + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java new file mode 100644 index 00000000..2fa636e7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java @@ -0,0 +1,75 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed; + + +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; + + +/******************************************************************************* + ** Definition for Streamed ETL process. + *******************************************************************************/ +public class StreamedETLProcess +{ + public static final String PROCESS_NAME = "etl.streamed"; + + public static final String FUNCTION_NAME_ETL = "streamedETL"; + + public static final String FIELD_SOURCE_TABLE = "sourceTable"; + public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + public static final String FIELD_MAPPING_JSON = "mappingJSON"; + public static final String FIELD_RECORD_COUNT = "recordCount"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QProcessMetaData defineProcessMetaData() + { + QStepMetaData etlFunction = new QBackendStepMetaData() + .withName(FUNCTION_NAME_ETL) + .withCode(new QCodeReference() + .withName(StreamedETLBackendStep.class.getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.BACKEND_STEP)) + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)) + .withField(new QFieldMetaData(FIELD_MAPPING_JSON, QFieldType.STRING)) + .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING))) + .withOutputMetaData(new QFunctionOutputMetaData() + .addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER))); + + return new QProcessMetaData() + .withName(PROCESS_NAME) + .addStep(etlFunction); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java index f29a62fa..71d2807f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java @@ -80,7 +80,7 @@ class ReportActionTest public void testBigger() throws Exception { // int recordCount = 2_000_000; // to really stress locally, use this. - int recordCount = 200_000; + int recordCount = 50_000; String filename = "/tmp/ReportActionTest.csv"; runReport(recordCount, filename, ReportFormat.CSV, false); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java index 99cbb6b6..049d3c57 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.modules.authentication; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException; @@ -31,7 +32,6 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.Auth0AuthenticationMetaData; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider; -import com.kingsrook.qqq.backend.core.state.StateProviderInterface; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0AuthenticationModule.AUTH0_ID_TOKEN_KEY; @@ -40,7 +40,8 @@ import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0Authent import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0AuthenticationModule.INVALID_TOKEN_ERROR; import static com.kingsrook.qqq.backend.core.modules.authentication.Auth0AuthenticationModule.TOKEN_NOT_PROVIDED_ERROR; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -49,9 +50,9 @@ import static org.junit.jupiter.api.Assertions.fail; *******************************************************************************/ public class Auth0AuthenticationModuleTest { - private static final String VALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA"; - private static final String INVALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA-thismakesinvalid"; - private static final String EXPIRED_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE4VDIxOjM4OjE1LjM4NloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MTgwNDc3LCJleHAiOjE2NTgyMTY0NzcsIm5vbmNlIjoiVkZkQlYzWmplR2hvY1cwMk9WZEtabHBLU0c1K1ZXbElhMEV3VkZaeFpVdEJVMDErZUZaT1RtMTNiZz09In0.fU7EwUgNrupOPz_PX_aQKON2xG1-LWD85xVo1Bn41WNEek-iMyJoch8l6NUihi7Bou14BoOfeWIG_sMqsLHqI2Pk7el7l1kigsjURx0wpiXadBt8piMxdIlxdToZEMuZCBzg7eJvXh4sM8tlV5cm0gPa6FT9Ih3VGJajNlXi5BcYS_JRpIvFvHn8-Bxj4KiAlZ5XPPkopjnDgP8kFfc4cMn_nxDkqWYlhj-5TaGW2xCLC9Qr_9UNxX0fm-CkKjYs3Z5ezbiXNkc-bxrCYvxeBeDPf8-T3EqrxCRVqCZSJ85BHdOc_E7UZC_g8bNj0umoplGwlCbzO4XIuOO-KlIaOg"; + private static final String VALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA"; + private static final String INVALID_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE5VDE2OjI0OjQ1LjgyMloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MjQ3OTAyLCJleHAiOjE2NTgyODM5MDIsIm5vbmNlIjoiZUhOdFMxbEtUR2N5ZG5KS1VVY3RkRTFVT0ZKNmJFNUxVVkEwZEdsRGVXOXZkVkl4UW41eVRrUlJlZz09In0.hib7JR8NDU2kx8Fj1bnzo3IUuabE6Hb-Z7HHZAJPQuF_Zdg3L1KDypn6SY7HAd_dsz2N8RkXfvQto-Y2g2ukuz7FxzNFgcVL99cyEO3YqmyCa6JTOTCrxdeaIE8QZpCEKvC28oeJBv0wO1Dwc--OVJMsK2vSzyxj1WNok64YYjWKLL4c0dFf-nj0KWFr1IU-tMiyWLDDiJw2Sa8M4YxXZYqdlkgNmrBPExgcm9l9SiT2l3Ts3Sgc_IyMVyMrnV8XX50EWdsm6vuCOSUcqf0XhjDQ7urZveoVwVLnYq3GcLhVBcy1Hr9RL8zPdPynOzsbX6uCww2Esrv6iwWrgQ5zBA-thismakesinvalid"; + private static final String EXPIRED_TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6IllrY2FkWTA0Q3RFVUFxQUdLNTk3ayJ9.eyJnaXZlbl9uYW1lIjoiVGltIiwiZmFtaWx5X25hbWUiOiJDaGFtYmVybGFpbiIsIm5pY2tuYW1lIjoidGltLmNoYW1iZXJsYWluIiwibmFtZSI6IlRpbSBDaGFtYmVybGFpbiIsInBpY3R1cmUiOiJodHRwczovL2xoMy5nb29nbGV1c2VyY29udGVudC5jb20vYS0vQUZkWnVjcXVSaUFvTzk1RG9URklnbUtseVA1akVBVnZmWXFnS0lHTkVubzE9czk2LWMiLCJsb2NhbGUiOiJlbiIsInVwZGF0ZWRfYXQiOiIyMDIyLTA3LTE4VDIxOjM4OjE1LjM4NloiLCJlbWFpbCI6InRpbS5jaGFtYmVybGFpbkBraW5nc3Jvb2suY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImlzcyI6Imh0dHBzOi8va2luZ3Nyb29rLnVzLmF1dGgwLmNvbS8iLCJzdWIiOiJnb29nbGUtb2F1dGgyfDEwODk2NDEyNjE3MjY1NzAzNDg2NyIsImF1ZCI6InNwQ1NtczAzcHpVZGRYN1BocHN4ZDlUd2FLMDlZZmNxIiwiaWF0IjoxNjU4MTgwNDc3LCJleHAiOjE2NTgyMTY0NzcsIm5vbmNlIjoiVkZkQlYzWmplR2hvY1cwMk9WZEtabHBLU0c1K1ZXbElhMEV3VkZaeFpVdEJVMDErZUZaT1RtMTNiZz09In0.fU7EwUgNrupOPz_PX_aQKON2xG1-LWD85xVo1Bn41WNEek-iMyJoch8l6NUihi7Bou14BoOfeWIG_sMqsLHqI2Pk7el7l1kigsjURx0wpiXadBt8piMxdIlxdToZEMuZCBzg7eJvXh4sM8tlV5cm0gPa6FT9Ih3VGJajNlXi5BcYS_JRpIvFvHn8-Bxj4KiAlZ5XPPkopjnDgP8kFfc4cMn_nxDkqWYlhj-5TaGW2xCLC9Qr_9UNxX0fm-CkKjYs3Z5ezbiXNkc-bxrCYvxeBeDPf8-T3EqrxCRVqCZSJ85BHdOc_E7UZC_g8bNj0umoplGwlCbzO4XIuOO-KlIaOg"; private static final String UNDECODABLE_TOKEN = "UNDECODABLE"; public static final String AUTH0_BASE_URL = "https://kingsrook.us.auth0.com/"; @@ -59,32 +60,79 @@ public class Auth0AuthenticationModuleTest /******************************************************************************* - ** Test a valid token where 'now' is set to a time that would be valid for it + ** Test an expired token where 'now' is set to a time that would not require it to be + ** re-checked, so it'll show as valid ** *******************************************************************************/ @Test - public void testLastTimeChecked() throws QAuthenticationException + public void testLastTimeCheckedNow() { - ////////////////////////////////////////////////////////// - // Tuesday, July 19, 2022 12:40:27.299 PM GMT-05:00 DST // - ////////////////////////////////////////////////////////// - Instant now = Instant.now(); + assertTrue(testLastTimeChecked(Instant.now(), UNDECODABLE_TOKEN), "A session just checked 'now' should always be valid"); + } - ///////////////////////////////////////////////////////// - // put the 'now' from the past into the state provider // - ///////////////////////////////////////////////////////// - StateProviderInterface spi = InMemoryStateProvider.getInstance(); - Auth0AuthenticationModule.Auth0StateKey key = new Auth0AuthenticationModule.Auth0StateKey(VALID_TOKEN); - spi.put(key, now); + + + /******************************************************************************* + ** Test an expired token where 'now' is set to a time that would not require it to be + ** re-checked, so it'll show as valid + ** + *******************************************************************************/ + @Test + public void testLastTimeCheckedJustUnderThreshold() + { + Instant underThreshold = Instant.now().minus(Auth0AuthenticationModule.ID_TOKEN_VALIDATION_INTERVAL_SECONDS - 60, ChronoUnit.SECONDS); + assertTrue(testLastTimeChecked(underThreshold, INVALID_TOKEN), "A session checked under threshold should be valid"); + } + + + + /******************************************************************************* + ** Test an expired token where 'now' is set to a time that would require it to be + ** re-checked + ** + *******************************************************************************/ + @Test + public void testLastTimeCheckedJustOverThreshold() + { + Instant overThreshold = Instant.now().minus(Auth0AuthenticationModule.ID_TOKEN_VALIDATION_INTERVAL_SECONDS + 60, ChronoUnit.SECONDS); + assertFalse(testLastTimeChecked(overThreshold, INVALID_TOKEN), "A session checked over threshold should be re-validated, and in this case, not be valid."); + } + + + + /******************************************************************************* + ** Test an expired token where 'now' is set to a time that would require it to be + ** re-checked + ** + *******************************************************************************/ + @Test + public void testLastTimeCheckedOverThresholdAndUndecodable() + { + Instant overThreshold = Instant.now().minus(Auth0AuthenticationModule.ID_TOKEN_VALIDATION_INTERVAL_SECONDS + 60, ChronoUnit.SECONDS); + assertFalse(testLastTimeChecked(overThreshold, UNDECODABLE_TOKEN), "A session checked over threshold should be re-validated, and in this case, not be valid."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean testLastTimeChecked(Instant lastTimeChecked, String token) + { + ///////////////////////////////////////////////////////////// + // put the input last-time-checked into the state provider // + ///////////////////////////////////////////////////////////// + Auth0AuthenticationModule.Auth0StateKey key = new Auth0AuthenticationModule.Auth0StateKey(token); + InMemoryStateProvider.getInstance().put(key, lastTimeChecked); ////////////////////// // build up session // ////////////////////// QSession session = new QSession(); - session.setIdReference(VALID_TOKEN); + session.setIdReference(token); Auth0AuthenticationModule auth0AuthenticationModule = new Auth0AuthenticationModule(); - assertEquals(true, auth0AuthenticationModule.isSessionValid(session), "Session should return as still valid."); + return (auth0AuthenticationModule.isSessionValid(getQInstance(), session)); } @@ -114,7 +162,7 @@ public class Auth0AuthenticationModuleTest /******************************************************************************* - ** Test failure case, token cant be decoded + ** Test failure case, token can't be decoded ** *******************************************************************************/ @Test @@ -220,4 +268,5 @@ public class Auth0AuthenticationModuleTest qInstance.setAuthentication(authenticationMetaData); return (qInstance); } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModuleTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModuleTest.java index 68c1e7d8..f775511f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModuleTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/FullyAnonymousAuthenticationModuleTest.java @@ -49,8 +49,8 @@ public class FullyAnonymousAuthenticationModuleTest assertNotNull(session.getIdReference(), "Session id ref should not be null"); assertNotNull(session.getUser(), "Session User should not be null"); assertNotNull(session.getUser().getIdReference(), "Session User id ref should not be null"); - assertTrue(fullyAnonymousAuthenticationModule.isSessionValid(session), "Any session should be valid"); - assertFalse(fullyAnonymousAuthenticationModule.isSessionValid(null), "null should be not valid"); + assertTrue(fullyAnonymousAuthenticationModule.isSessionValid(null, session), "Any session should be valid"); + assertFalse(fullyAnonymousAuthenticationModule.isSessionValid(null, null), "null should be not valid"); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcessTest.java new file mode 100644 index 00000000..97d756c9 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcessTest.java @@ -0,0 +1,89 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed; + + +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for BasicETLProcess + *******************************************************************************/ +class StreamedETLProcessTest +{ + + /******************************************************************************* + ** Simplest happy path + *******************************************************************************/ + @Test + public void test() throws QException + { + RunProcessInput request = new RunProcessInput(TestUtils.defineInstance()); + request.setSession(TestUtils.getMockSession()); + request.setProcessName(StreamedETLProcess.PROCESS_NAME); + request.addValue(StreamedETLProcess.FIELD_SOURCE_TABLE, TestUtils.defineTablePerson().getName()); + request.addValue(StreamedETLProcess.FIELD_DESTINATION_TABLE, TestUtils.definePersonFileTable().getName()); + request.addValue(StreamedETLProcess.FIELD_MAPPING_JSON, ""); + + RunProcessOutput result = new RunProcessAction().execute(request); + assertNotNull(result); + assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process"); + assertTrue(result.getException().isEmpty()); + } + + + + /******************************************************************************* + ** Basic example of doing a mapping transformation + *******************************************************************************/ + @Test + public void testMappingTransformation() throws QException + { + RunProcessInput request = new RunProcessInput(TestUtils.defineInstance()); + request.setSession(TestUtils.getMockSession()); + request.setProcessName(StreamedETLProcess.PROCESS_NAME); + request.addValue(StreamedETLProcess.FIELD_SOURCE_TABLE, TestUtils.definePersonFileTable().getName()); + request.addValue(StreamedETLProcess.FIELD_DESTINATION_TABLE, TestUtils.defineTableIdAndNameOnly().getName()); + + /////////////////////////////////////////////////////////////////////////////////////// + // define our mapping from destination-table field names to source-table field names // + /////////////////////////////////////////////////////////////////////////////////////// + QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping().withMapping("name", "firstName"); + // request.addValue(StreamedETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping.getMapping())); + request.addValue(StreamedETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping)); + + RunProcessOutput result = new RunProcessAction().execute(request); + assertNotNull(result); + assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process"); + assertTrue(result.getException().isEmpty()); + } + +} \ No newline at end of file From 54103f47adb654df1e3bc9050f5745638100eb65 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 4 Aug 2022 11:35:17 -0500 Subject: [PATCH 06/18] Changes pushed to qqq-backend-module-rdbms (solo-repo) in 0.2 support --- .../rdbms/actions/RDBMSInsertAction.java | 47 +++++- .../rdbms/actions/RDBMSTransaction.java | 141 ++++++++++++++++++ .../module/rdbms/jdbc/QueryManager.java | 22 +++ 3 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransaction.java diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index 3c32d746..b9e76476 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -28,6 +28,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; @@ -92,7 +93,20 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte List outputRecords = new ArrayList<>(); rs.setRecords(outputRecords); - try(Connection connection = getConnection(insertInput)) + Connection connection; + boolean needToCloseConnection = false; + if(insertInput.getTransaction() != null && insertInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction) + { + LOG.debug("Using connection from insertInput [" + rdbmsTransaction.getConnection() + "]"); + connection = rdbmsTransaction.getConnection(); + } + else + { + connection = getConnection(insertInput); + needToCloseConnection = true; + } + + try { for(List page : CollectionUtils.getPages(insertInput.getRecords(), QueryManager.PAGE_SIZE)) { @@ -130,6 +144,13 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte } } } + finally + { + if(needToCloseConnection) + { + connection.close(); + } + } return rs; } @@ -139,4 +160,28 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte } } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QBackendTransaction openTransaction(InsertInput insertInput) throws QException + { + try + { + LOG.info("Opening transaction"); + Connection connection = getConnection(insertInput); + connection.setAutoCommit(false); + System.out.println("Set connection [" + connection + "] to auto-commit false"); + + return (new RDBMSTransaction(connection)); + } + catch(Exception e) + { + throw new QException("Error opening transaction: " + e.getMessage(), e); + } + } + + } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransaction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransaction.java new file mode 100644 index 00000000..529f69ff --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransaction.java @@ -0,0 +1,141 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.actions; + + +import java.io.IOException; +import java.sql.Connection; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** RDBMS implementation of backend transaction. + ** + ** Stores a jdbc connection, which is set to autoCommit(false). + *******************************************************************************/ +public class RDBMSTransaction extends QBackendTransaction +{ + private static final Logger LOG = LogManager.getLogger(RDBMSTransaction.class); + + private Connection connection; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public RDBMSTransaction(Connection connection) + { + this.connection = connection; + } + + + + /******************************************************************************* + ** Getter for connection + ** + *******************************************************************************/ + public Connection getConnection() + { + return connection; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void commit() throws QException + { + try + { + RDBMSTransaction.LOG.info("Committing transaction"); + System.out.println("Calling commit on connection [" + connection + "]"); + connection.commit(); + RDBMSTransaction.LOG.info("Commit complete"); + } + catch(Exception e) + { + RDBMSTransaction.LOG.error("Error committing transaction", e); + throw new QException("Error committing transaction: " + e.getMessage(), e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void rollback() throws QException + { + try + { + RDBMSTransaction.LOG.info("Rolling back transaction"); + connection.rollback(); + RDBMSTransaction.LOG.info("Rollback complete"); + } + catch(Exception e) + { + RDBMSTransaction.LOG.error("Error rolling back transaction", e); + throw new QException("Error rolling back transaction: " + e.getMessage(), e); + } + } + + + + /******************************************************************************* + * Closes this stream and releases any system resources associated + * with it. If the stream is already closed then invoking this + * method has no effect. + * + *

As noted in {@link AutoCloseable#close()}, cases where the + * close may fail require careful attention. It is strongly advised + * to relinquish the underlying resources and to internally + * mark the {@code Closeable} as closed, prior to throwing + * the {@code IOException}. + * + * @throws IOException + * if an I/O error occurs + *******************************************************************************/ + @Override + public void close() + { + try + { + if(connection.isClosed()) + { + return; + } + + connection.close(); + } + catch(Exception e) + { + LOG.error("Error closing connection - possible jdbc connection leak", e); + } + } +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java index fbd634e5..47e25f6f 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java @@ -685,6 +685,11 @@ public class QueryManager bindParam(statement, index, l.intValue()); return (1); } + else if(value instanceof Double d) + { + bindParam(statement, index, d.doubleValue()); + return (1); + } else if(value instanceof String s) { bindParam(statement, index, s); @@ -851,6 +856,23 @@ public class QueryManager + /******************************************************************************* + * + *******************************************************************************/ + public static void bindParam(PreparedStatement statement, int index, Double value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.DOUBLE); + } + else + { + statement.setDouble(index, value); + } + } + + + /******************************************************************************* * *******************************************************************************/ From 2bcf0b58a9dac55752904282b86bdb0c2724a2a8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 4 Aug 2022 13:20:07 -0500 Subject: [PATCH 07/18] Changes pushed to qqq-backend-module-filesystem (solo-repo) in 0.2 support --- .../actions/AbstractBaseFilesystemAction.java | 30 ++-- .../basic/BasicETLCleanupSourceFilesStep.java | 2 + .../StreamedETLFilesystemBackendStep.java | 75 ++++++++++ .../module/filesystem/s3/utils/S3Utils.java | 4 +- .../backend/module/filesystem/TestUtils.java | 134 +++++++++++++++--- .../local/FilesystemBackendModuleTest.java | 8 +- .../local/actions/FilesystemActionTest.java | 37 ++++- .../actions/FilesystemQueryActionTest.java | 2 +- .../FilesystemBackendMetaDataTest.java | 6 +- .../BasicETLCleanupSourceFilesStepTest.java | 4 +- .../StreamedETLFilesystemBackendStepTest.java | 62 ++++++++ .../s3/actions/S3QueryActionTest.java | 3 +- .../model/metadata/S3BackendMetaDataTest.java | 11 +- 13 files changed, 327 insertions(+), 51 deletions(-) create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStep.java create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStepTest.java diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 84341c17..02b26622 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -201,10 +201,16 @@ public abstract class AbstractBaseFilesystemAction String fileContents = IOUtils.toString(readFile(file)); fileContents = customizeFileContentsAfterReading(table, fileContents); - List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); - addBackendDetailsToRecords(recordsInFile, file); - - queryOutput.addRecords(recordsInFile); + if(queryInput.getRecordPipe() != null) + { + new CsvToQRecordAdapter().buildRecordsFromCsv(queryInput.getRecordPipe(), fileContents, table, null, (record -> addBackendDetailsToRecord(record, file))); + } + else + { + List recordsInFile = new CsvToQRecordAdapter().buildRecordsFromCsv(fileContents, table, null); + addBackendDetailsToRecords(recordsInFile, file); + queryOutput.addRecords(recordsInFile); + } break; } case JSON: @@ -212,6 +218,7 @@ public abstract class AbstractBaseFilesystemAction String fileContents = IOUtils.toString(readFile(file)); fileContents = customizeFileContentsAfterReading(table, fileContents); + // todo - pipe support!! List recordsInFile = new JsonToQRecordAdapter().buildRecordsFromJson(fileContents, table, null); addBackendDetailsToRecords(recordsInFile, file); @@ -241,10 +248,17 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ protected void addBackendDetailsToRecords(List recordsInFile, FILE file) { - recordsInFile.forEach(record -> - { - record.withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file)); - }); + recordsInFile.forEach(r -> addBackendDetailsToRecord(r, file)); + } + + + + /******************************************************************************* + ** Add backend details to a record about the file that it is in. + *******************************************************************************/ + protected void addBackendDetailsToRecord(QRecord record, FILE file) + { + record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, getFullPathForFile(file)); } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java index f455f018..7add637a 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java @@ -94,11 +94,13 @@ public class BasicETLCleanupSourceFilesStep implements BackendStep String moveOrDelete = runBackendStepInput.getValueString(FIELD_MOVE_OR_DELETE); if(VALUE_DELETE.equals(moveOrDelete)) { + LOG.info("Deleting ETL source file: " + sourceFile); actionBase.deleteFile(runBackendStepInput.getInstance(), table, sourceFile); } else if(VALUE_MOVE.equals(moveOrDelete)) { String destinationForMoves = runBackendStepInput.getValueString(FIELD_DESTINATION_FOR_MOVES); + LOG.info("Moving ETL source file: " + sourceFile + " to " + destinationForMoves); if(!StringUtils.hasContent(destinationForMoves)) { throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value.")); diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStep.java new file mode 100644 index 00000000..8a255d88 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStep.java @@ -0,0 +1,75 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed; + + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLBackendStep; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic.BasicETLCollectSourceFileNamesStep; + + +/******************************************************************************* + ** Extension to the base StreamedETLBackendStep, unique for where the source + ** table is a filesystem, where we want/need to collect the filenames that were + ** processed in the Extract step, so they can be passed into the cleanup step. + ** + ** Similar in purpose to the BasicETLCollectSourceFileNamesStep - only in this + ** case, due to the streaming behavior of the StreamedETLProcess, we can't really + ** inject this code as a separate backend step - so instead we extend that step, + ** and override its postTransform method to intercept the records & file names. + *******************************************************************************/ +public class StreamedETLFilesystemBackendStep extends StreamedETLBackendStep +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected void preTransform(List qRecords, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + Set sourceFiles = qRecords.stream() + .map(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH)) + .collect(Collectors.toSet()); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // expect that we'll be called on multiple "pages" of records as they run through the pipe. // + // each time we're called, we need to: // + // - get the unique file paths in this list of records // + // - if we previously set the list of file names in the output, then split that value up and add those names to the set we see now // + // - set the list of name (joined by commas) in the output // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + String existingListOfFileNames = runBackendStepOutput.getValueString(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS); + if(existingListOfFileNames != null) + { + sourceFiles.addAll(List.of(existingListOfFileNames.split(","))); + } + runBackendStepOutput.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", sourceFiles)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java index b791d355..9b537f95 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java @@ -137,7 +137,7 @@ public class S3Utils //////////////////////////////////////////////////////////////////////////////// if(key.endsWith("/")) { - LOG.debug("Skipping file [{}] because it is a folder", key); + // LOG.debug("Skipping file [{}] because it is a folder", key); continue; } @@ -146,7 +146,7 @@ public class S3Utils /////////////////////////////////////////// if(!pathMatcher.matches(Path.of(URI.create("file:///" + key)))) { - LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob); + // LOG.debug("Skipping file [{}] that does not match glob [{}]", key, glob); continue; } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index fd7d0714..103df995 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -24,20 +24,27 @@ package com.kingsrook.qqq.backend.module.filesystem; import java.io.File; import java.io.IOException; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed.StreamedETLFilesystemBackendStep; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; @@ -49,10 +56,16 @@ import org.apache.commons.io.FileUtils; *******************************************************************************/ public class TestUtils { - public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; - public static final String BACKEND_NAME_S3 = "s3"; - public static final String TABLE_NAME_PERSON_LOCAL_FS = "person"; - public static final String TABLE_NAME_PERSON_S3 = "person-s3"; + public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; + public static final String BACKEND_NAME_S3 = "s3"; + public static final String BACKEND_NAME_MOCK = "mock"; + + public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json"; + public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv"; + public static final String TABLE_NAME_PERSON_S3 = "person-s3"; + public static final String TABLE_NAME_PERSON_MOCK = "person-mock"; + + public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed"; /////////////////////////////////////////////////////////////////// // shouldn't be accessed directly, as we append a counter to it. // @@ -112,14 +125,18 @@ public class TestUtils /******************************************************************************* ** *******************************************************************************/ - public static QInstance defineInstance() throws QInstanceValidationException + public static QInstance defineInstance() throws QException { QInstance qInstance = new QInstance(); qInstance.setAuthentication(defineAuthentication()); qInstance.addBackend(defineLocalFilesystemBackend()); qInstance.addTable(defineLocalFilesystemJSONPersonTable()); + qInstance.addTable(defineLocalFilesystemCSVPersonTable()); qInstance.addBackend(defineS3Backend()); qInstance.addTable(defineS3CSVPersonTable()); + qInstance.addBackend(defineMockBackend()); + qInstance.addTable(defineMockPersonTable()); + qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); new QInstanceValidator().validate(qInstance); @@ -159,21 +176,55 @@ public class TestUtils public static QTableMetaData defineLocalFilesystemJSONPersonTable() { return new QTableMetaData() - .withName(TABLE_NAME_PERSON_LOCAL_FS) + .withName(TABLE_NAME_PERSON_LOCAL_FS_JSON) .withLabel("Person") .withBackendName(defineLocalFilesystemBackend().getName()) .withPrimaryKeyField("id") - .withField(new QFieldMetaData("id", QFieldType.INTEGER)) - .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) - .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) - .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name")) - .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name")) - .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) - .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withFields(defineCommonPersonTableFields()) .withBackendDetails(new FilesystemTableBackendDetails() .withBasePath("persons") .withRecordFormat(RecordFormat.JSON) .withCardinality(Cardinality.MANY) + .withGlob("*.json") + ); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static List defineCommonPersonTableFields() + { + return (List.of( + new QFieldMetaData("id", QFieldType.INTEGER), + new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date"), + new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date"), + new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name"), + new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name"), + new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date"), + new QFieldMetaData("email", QFieldType.STRING) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineLocalFilesystemCSVPersonTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSON_LOCAL_FS_CSV) + .withLabel("Person") + .withBackendName(defineLocalFilesystemBackend().getName()) + .withPrimaryKeyField("id") + .withFields(defineCommonPersonTableFields()) + .withBackendDetails(new FilesystemTableBackendDetails() + .withBasePath("persons-csv") + .withRecordFormat(RecordFormat.CSV) + .withCardinality(Cardinality.MANY) + .withGlob("*.csv") ); } @@ -202,13 +253,7 @@ public class TestUtils .withLabel("Person S3 Table") .withBackendName(defineS3Backend().getName()) .withPrimaryKeyField("id") - .withField(new QFieldMetaData("id", QFieldType.INTEGER)) - .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) - .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) - .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name")) - .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name")) - .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) - .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withFields(defineCommonPersonTableFields()) .withBackendDetails(new S3TableBackendDetails() .withRecordFormat(RecordFormat.CSV) .withCardinality(Cardinality.MANY) @@ -220,7 +265,52 @@ public class TestUtils /******************************************************************************* ** *******************************************************************************/ - public static QSession getMockSession() throws QInstanceValidationException + public static QBackendMetaData defineMockBackend() + { + return (new QBackendMetaData() + .withBackendType("mock") + .withName(BACKEND_NAME_MOCK)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineMockPersonTable() + { + return (new QTableMetaData() + .withName(TABLE_NAME_PERSON_MOCK) + .withLabel("Person Mock Table") + .withBackendName(BACKEND_NAME_MOCK) + .withPrimaryKeyField("id") + .withFields(defineCommonPersonTableFields())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QProcessMetaData defineStreamedLocalCsvToMockETLProcess() throws QException + { + QProcessMetaData qProcessMetaData = new StreamedETLProcess().defineProcessMetaData(); + qProcessMetaData.setName(PROCESS_NAME_STREAMED_ETL); + QBackendStepMetaData backendStep = qProcessMetaData.getBackendStep(StreamedETLProcess.FUNCTION_NAME_ETL); + backendStep.setCode(new QCodeReference(StreamedETLFilesystemBackendStep.class)); + + backendStep.getInputMetaData().getFieldThrowing(StreamedETLProcess.FIELD_SOURCE_TABLE).setDefaultValue(TABLE_NAME_PERSON_LOCAL_FS_CSV); + backendStep.getInputMetaData().getFieldThrowing(StreamedETLProcess.FIELD_DESTINATION_TABLE).setDefaultValue(TABLE_NAME_PERSON_MOCK); + + return (qProcessMetaData); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QSession getMockSession() throws QException { MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule(); return (mockAuthenticationModule.createSession(defineInstance(), null)); diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java index 17318fd3..ea52bc97 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -70,7 +70,7 @@ public class FilesystemBackendModuleTest public void testDeleteFile() throws Exception { QInstance qInstance = TestUtils.defineInstance(); - QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON); ///////////////////////////////////////////////////////////////////////////////////////////// // first list the files - then delete one, then re-list, and assert that we have one fewer // @@ -94,7 +94,7 @@ public class FilesystemBackendModuleTest public void testDeleteFileDoesNotExist() throws Exception { QInstance qInstance = TestUtils.defineInstance(); - QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // first list the files - then try to delete a fake path, then re-list, and assert that we have the same count // @@ -120,7 +120,7 @@ public class FilesystemBackendModuleTest public void testMoveFile() throws Exception { QInstance qInstance = TestUtils.defineInstance(); - QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON); String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath(); String subPath = basePath + File.separator + "subdir"; @@ -157,7 +157,7 @@ public class FilesystemBackendModuleTest public void testMoveFileDoesNotExit() throws Exception { QInstance qInstance = TestUtils.defineInstance(); - QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON); String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath(); String subPath = basePath + File.separator + "subdir"; List filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java index 755ab477..94cbfbf7 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java @@ -80,7 +80,8 @@ public class FilesystemActionTest fail("Failed to make directories at [" + baseDirectory + "] for filesystem backend module"); } - writePersonFiles(baseDirectory); + writePersonJSONFiles(baseDirectory); + writePersonCSVFiles(baseDirectory); } @@ -88,7 +89,7 @@ public class FilesystemActionTest /******************************************************************************* ** Write some data files into the directory for the filesystem module. *******************************************************************************/ - private void writePersonFiles(File baseDirectory) throws IOException + private void writePersonJSONFiles(File baseDirectory) throws IOException { String fullPath = baseDirectory.getAbsolutePath(); if (TestUtils.defineLocalFilesystemJSONPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) @@ -118,6 +119,38 @@ public class FilesystemActionTest + /******************************************************************************* + ** Write some data files into the directory for the filesystem module. + *******************************************************************************/ + private void writePersonCSVFiles(File baseDirectory) throws IOException + { + String fullPath = baseDirectory.getAbsolutePath(); + if (TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details) + { + if (StringUtils.hasContent(details.getBasePath())) + { + fullPath += File.separatorChar + details.getBasePath(); + } + } + fullPath += File.separatorChar; + + String csvData1 = """ + "id","createDate","modifyDate","firstName","lastName","birthDate","email" + "1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com" + "2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com" + """; + FileUtils.writeStringToFile(new File(fullPath + "FILE-1.csv"), csvData1); + + String csvData2 = """ + "id","createDate","modifyDate","firstName","lastName","birthDate","email" + "3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com" + "4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com" + "5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com\""""; // intentionally no \n at EOL here + FileUtils.writeStringToFile(new File(fullPath + "FILE-2.csv"), csvData2); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java index cc90a4bc..90771e43 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java @@ -71,7 +71,7 @@ public class FilesystemQueryActionTest extends FilesystemActionTest QueryInput queryInput = new QueryInput(); QInstance instance = TestUtils.defineInstance(); - QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON); table.withCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ, new QCodeReference() .withName(ValueUpshifter.class.getName()) .withCodeType(QCodeType.JAVA) diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java index e293c89a..9f9780b9 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java @@ -24,7 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.local.model.metadata; import java.io.IOException; import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; @@ -44,7 +44,7 @@ class FilesystemBackendMetaDataTest ** Test that an instance can be serialized as expected *******************************************************************************/ @Test - public void testSerializingToJson() throws QInstanceValidationException + public void testSerializingToJson() throws QException { TestUtils.resetTestInstanceCounter(); QInstance qInstance = TestUtils.defineInstance(); @@ -62,7 +62,7 @@ class FilesystemBackendMetaDataTest ** Test that an instance can be deserialized as expected *******************************************************************************/ @Test - public void testDeserializingFromJson() throws IOException, QInstanceValidationException + public void testDeserializingFromJson() throws IOException, QException { QInstanceAdapter qInstanceAdapter = new QInstanceAdapter(); diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java index 3a43066a..33604e1b 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java @@ -198,7 +198,7 @@ public class BasicETLCleanupSourceFilesStepTest runBackendStepInput.setProcessName(qProcessMetaData.getName()); // runFunctionRequest.setRecords(records); runBackendStepInput.setSession(TestUtils.getMockSession()); - runBackendStepInput.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + runBackendStepInput.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON); runBackendStepInput.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3); runBackendStepInput.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet)); @@ -219,7 +219,7 @@ public class BasicETLCleanupSourceFilesStepTest private String getRandomFilePathPersonTable(QInstance qInstance) { FilesystemBackendMetaData backend = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS); - FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails(); + FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON).getBackendDetails(); String tablePath = backend.getBasePath() + File.separator + backendDetails.getBasePath(); String filePath = tablePath + File.separator + UUID.randomUUID(); return filePath; diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStepTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStepTest.java new file mode 100644 index 00000000..2c71d2a2 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/streamed/StreamedETLFilesystemBackendStepTest.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed; + + +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest; +import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic.BasicETLCollectSourceFileNamesStep; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for StreamedETLFilesystemBackendStep + *******************************************************************************/ +class StreamedETLFilesystemBackendStepTest extends FilesystemActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void runFullProcess() throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + + RunProcessInput runProcessInput = new RunProcessInput(qInstance); + runProcessInput.setSession(TestUtils.getMockSession()); + runProcessInput.setProcessName(TestUtils.PROCESS_NAME_STREAMED_ETL); + + RunProcessOutput output = new RunProcessAction().execute(runProcessInput); + String sourceFilePaths = ValueUtils.getValueAsString(output.getValues().get(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS)); + assertThat(sourceFilePaths) + .contains("FILE-1.csv") + .contains("FILE-2.csv"); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java index ef84e292..3f790fa0 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java @@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; @@ -60,7 +59,7 @@ public class S3QueryActionTest extends BaseS3Test /******************************************************************************* ** *******************************************************************************/ - private QueryInput initQueryRequest() throws QInstanceValidationException + private QueryInput initQueryRequest() throws QException { QueryInput queryInput = new QueryInput(); queryInput.setInstance(TestUtils.defineInstance()); diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java index 001fc6cb..d0d2e4b2 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java @@ -22,9 +22,8 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata; -import java.io.IOException; import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter; -import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.module.filesystem.TestUtils; @@ -44,7 +43,7 @@ class S3BackendMetaDataTest ** Test that an instance can be serialized as expected *******************************************************************************/ @Test - public void testSerializingToJson() throws QInstanceValidationException + public void testSerializingToJson() throws QException { TestUtils.resetTestInstanceCounter(); QInstance qInstance = TestUtils.defineInstance(); @@ -62,7 +61,7 @@ class S3BackendMetaDataTest ** Test that an instance can be deserialized as expected *******************************************************************************/ @Test - public void testDeserializingFromJson() throws IOException, QInstanceValidationException + public void testDeserializingFromJson() throws Exception { QInstanceAdapter qInstanceAdapter = new QInstanceAdapter(); @@ -71,6 +70,8 @@ class S3BackendMetaDataTest QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); assertThat(deserialized.getBackends()).usingRecursiveComparison() + // TODO seeing occassional flaps on this field - where it can be null 1 out of 10 runs... unclear why. + // .ignoringFields("mock.backendType") .isEqualTo(qInstance.getBackends()); } -} \ No newline at end of file +} From 9839cafdd504f5ec40a123f0b4e3c4c33580afc9 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 4 Aug 2022 13:20:47 -0500 Subject: [PATCH 08/18] Changes pushed to qqq-middleware-javalin (solo-repo) in 0.2 support --- .../javalin/QJavalinImplementation.java | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 2dddf8d6..b67241ee 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -251,11 +251,39 @@ public class QJavalinImplementation try { Map authenticationContext = new HashMap<>(); - authenticationContext.put(SESSION_ID_COOKIE_NAME, context.cookie(SESSION_ID_COOKIE_NAME)); + + ///////////////////////////////////////////////////////////////////////////////// + // look for a token in either the sessionId cookie, or an Authorization header // + ///////////////////////////////////////////////////////////////////////////////// + String sessionIdCookieValue = context.cookie(SESSION_ID_COOKIE_NAME); + if(StringUtils.hasContent(sessionIdCookieValue)) + { + authenticationContext.put(SESSION_ID_COOKIE_NAME, sessionIdCookieValue); + } + else + { + String authorizationHeaderValue = context.header("Authorization"); + if (authorizationHeaderValue != null) + { + String bearerPrefix = "Bearer "; + if(authorizationHeaderValue.startsWith(bearerPrefix)) + { + authorizationHeaderValue = authorizationHeaderValue.replaceFirst(bearerPrefix, ""); + } + authenticationContext.put(SESSION_ID_COOKIE_NAME, authorizationHeaderValue); + } + } + QSession session = authenticationModule.createSession(qInstance, authenticationContext); input.setSession(session); - context.cookie(SESSION_ID_COOKIE_NAME, session.getIdReference(), SESSION_COOKIE_AGE); + ///////////////////////////////////////////////////////////////////////////////// + // if we got a session id cookie in, then send it back with updated cookie age // + ///////////////////////////////////////////////////////////////////////////////// + if(StringUtils.hasContent(sessionIdCookieValue)) + { + context.cookie(SESSION_ID_COOKIE_NAME, session.getIdReference(), SESSION_COOKIE_AGE); + } } catch(QAuthenticationException qae) { From 2f937b9436539e13a3d865b40814a61ec92e5e87 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 4 Aug 2022 13:22:22 -0500 Subject: [PATCH 09/18] Adding missed commit from previous --- .gitignore | 1 + .../core/model/actions/processes/RunBackendStepOutput.java | 2 +- .../java/com/kingsrook/qqq/backend/core/utils/TestUtils.java | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 64c6c21c..e6a143b9 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ target/ hs_err_pid* .DS_Store *.swp +.flattened-pom.xml diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java index a6460bff..80e5d531 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java @@ -35,7 +35,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; ** Output data container for the RunBackendStep action ** *******************************************************************************/ -public class RunBackendStepOutput extends AbstractActionOutput +public class RunBackendStepOutput extends AbstractActionOutput implements Serializable { private ProcessState processState; private Exception exception; // todo - make optional diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 6bae6380..9c82df81 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -54,6 +54,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; /******************************************************************************* @@ -85,6 +86,7 @@ public class TestUtils qInstance.addProcess(defineProcessGreetPeopleInteractive()); qInstance.addProcess(defineProcessAddToPeoplesAge()); qInstance.addProcess(new BasicETLProcess().defineProcessMetaData()); + qInstance.addProcess(new StreamedETLProcess().defineProcessMetaData()); System.out.println(new QInstanceAdapter().qInstanceToJson(qInstance)); From 3c12979571cf6d41789b14aa829afd10ca952098 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 4 Aug 2022 13:36:25 -0500 Subject: [PATCH 10/18] Updated to ignore field that is sometimes flapping :( --- .../local/model/metadata/FilesystemBackendMetaDataTest.java | 4 +++- .../filesystem/s3/model/metadata/S3BackendMetaDataTest.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java index 9f9780b9..e67936d5 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaDataTest.java @@ -71,6 +71,8 @@ class FilesystemBackendMetaDataTest QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); assertThat(deserialized.getBackends()).usingRecursiveComparison() + // TODO seeing occassional flaps on this field - where it can be null 1 out of 10 runs... unclear why. + .ignoringFields("mock.backendType") .isEqualTo(qInstance.getBackends()); } -} \ No newline at end of file +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java index d0d2e4b2..39103602 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaDataTest.java @@ -71,7 +71,7 @@ class S3BackendMetaDataTest QInstance deserialized = qInstanceAdapter.jsonToQInstanceIncludingBackends(json); assertThat(deserialized.getBackends()).usingRecursiveComparison() // TODO seeing occassional flaps on this field - where it can be null 1 out of 10 runs... unclear why. - // .ignoringFields("mock.backendType") + .ignoringFields("mock.backendType") .isEqualTo(qInstance.getBackends()); } } From 19afc0fc1011999346d33811a82abb56554ffc59 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 4 Aug 2022 16:51:03 -0500 Subject: [PATCH 11/18] Adding apps; starting field displayFormats --- .gitignore | 1 + .../core/actions/metadata/MetaDataAction.java | 72 ++++- .../core/actions/values/QValueFormatter.java | 75 +++++ .../core/instances/QInstanceEnricher.java | 19 ++ .../core/instances/QInstanceValidator.java | 238 +++++++++++---- .../model/actions/AbstractActionInput.java | 2 +- .../actions/metadata/MetaDataOutput.java | 50 +++ .../qqq/backend/core/model/data/QRecord.java | 36 +++ .../core/model/metadata/QInstance.java | 73 ++++- .../model/metadata/fields/DisplayFormat.java | 40 +++ .../model/metadata/fields/QFieldMetaData.java | 33 ++ .../model/metadata/fields/QFieldType.java | 4 + .../metadata/frontend/AppTreeNodeType.java | 33 ++ .../frontend/QFrontendAppMetaData.java | 164 ++++++++++ .../metadata/layout/QAppChildMetaData.java | 58 ++++ .../model/metadata/layout/QAppMetaData.java | 237 +++++++++++++++ .../core/model/metadata/layout/QIcon.java | 91 ++++++ .../metadata/processes/QProcessMetaData.java | 30 +- .../model/metadata/tables/QTableMetaData.java | 74 ++++- .../implementations/mock/MockQueryAction.java | 1 + .../actions/metadata/MetaDataActionTest.java | 67 +++- .../core/actions/tables/QueryActionTest.java | 10 + .../instances/QInstanceValidatorTest.java | 285 +++++++++++------- .../qqq/backend/core/utils/TestUtils.java | 46 ++- .../javalin/QJavalinImplementation.java | 7 +- .../qqq/backend/javalin/QJavalinTestBase.java | 3 +- .../sampleapp/SampleJavalinServer.java | 1 + .../sampleapp/SampleMetaDataProvider.java | 60 +++- 28 files changed, 1624 insertions(+), 186 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DisplayFormat.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNodeType.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppChildMetaData.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QIcon.java diff --git a/.gitignore b/.gitignore index 64c6c21c..e6a143b9 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ target/ hs_err_pid* .DS_Store *.swp +.flattened-pom.xml diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java index 4f933dd8..f15e319c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java @@ -22,14 +22,19 @@ package com.kingsrook.qqq.backend.core.actions.metadata; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -50,22 +55,87 @@ public class MetaDataAction // todo pre-customization - just get to modify the request? MetaDataOutput metaDataOutput = new MetaDataOutput(); + Map treeNodes = new LinkedHashMap<>(); + + ///////////////////////////////////// + // map tables to frontend metadata // + ///////////////////////////////////// Map tables = new LinkedHashMap<>(); for(Map.Entry entry : metaDataInput.getInstance().getTables().entrySet()) { tables.put(entry.getKey(), new QFrontendTableMetaData(entry.getValue(), false)); + treeNodes.put(entry.getKey(), new QFrontendAppMetaData(entry.getValue())); } metaDataOutput.setTables(tables); + //////////////////////////////////////// + // map processes to frontend metadata // + //////////////////////////////////////// Map processes = new LinkedHashMap<>(); for(Map.Entry entry : metaDataInput.getInstance().getProcesses().entrySet()) { processes.put(entry.getKey(), new QFrontendProcessMetaData(entry.getValue(), false)); + treeNodes.put(entry.getKey(), new QFrontendAppMetaData(entry.getValue())); } metaDataOutput.setProcesses(processes); - // todo post-customization - can do whatever w/ the result if you want + /////////////////////////////////// + // map apps to frontend metadata // + /////////////////////////////////// + Map apps = new LinkedHashMap<>(); + for(Map.Entry entry : metaDataInput.getInstance().getApps().entrySet()) + { + apps.put(entry.getKey(), new QFrontendAppMetaData(entry.getValue())); + treeNodes.put(entry.getKey(), new QFrontendAppMetaData(entry.getValue())); + + for(QAppChildMetaData child : entry.getValue().getChildren()) + { + apps.get(entry.getKey()).addChild(new QFrontendAppMetaData(child)); + } + } + metaDataOutput.setApps(apps); + + //////////////////////////////////////////////// + // organize app tree nodes by their hierarchy // + //////////////////////////////////////////////// + List appTree = new ArrayList<>(); + for(QAppMetaData appMetaData : metaDataInput.getInstance().getApps().values()) + { + if(appMetaData.getParentAppName() == null) + { + buildAppTree(treeNodes, appTree, appMetaData); + } + } + metaDataOutput.setAppTree(appTree); + + // todo post-customization - can do whatever w/ the result if you want? return metaDataOutput; } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void buildAppTree(Map treeNodes, List nodeList, QAppChildMetaData childMetaData) + { + QFrontendAppMetaData treeNode = treeNodes.get(childMetaData.getName()); + if(treeNode == null) + { + return; + } + + nodeList.add(treeNode); + if(childMetaData instanceof QAppMetaData app) + { + if(app.getChildren() != null) + { + for(QAppChildMetaData child : app.getChildren()) + { + buildAppTree(treeNodes, treeNode.getChildren(), child); + } + } + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java new file mode 100644 index 00000000..107b66bd --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -0,0 +1,75 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.values; + + +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Utility to apply display formats to values for fields + *******************************************************************************/ +public class QValueFormatter +{ + private static final Logger LOG = LogManager.getLogger(QValueFormatter.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String formatValue(QFieldMetaData field, Serializable value) + { + ////////////////////////////////// + // null values get null results // + ////////////////////////////////// + if(value == null) + { + return (null); + } + + //////////////////////////////////////////////////////// + // if the field has a display format, try to apply it // + //////////////////////////////////////////////////////// + if(StringUtils.hasContent(field.getDisplayFormat())) + { + try + { + return (field.getDisplayFormat().formatted(value)); + } + catch(Exception e) + { + LOG.warn("Error formatting value [" + value + "] for field [" + field.getName() + "] with format [" + field.getDisplayFormat() + "]: " + e.getMessage()); + } + } + + //////////////////////////////////////// + // by default, just get back a string // + //////////////////////////////////////// + return (ValueUtils.getValueAsString(value)); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 569ce603..de16582d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; @@ -77,6 +78,11 @@ public class QInstanceEnricher { qInstance.getBackends().values().forEach(this::enrich); } + + if(qInstance.getApps() != null) + { + qInstance.getApps().values().forEach(this::enrich); + } } @@ -172,6 +178,19 @@ public class QInstanceEnricher + /******************************************************************************* + ** + *******************************************************************************/ + private void enrich(QAppMetaData app) + { + if(!StringUtils.hasContent(app.getLabel())) + { + app.setLabel(nameToLabel(app.getName())); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index f1e018ff..c7665022 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -23,10 +23,15 @@ package com.kingsrook.qqq.backend.core.instances; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -76,60 +81,10 @@ public class QInstanceValidator List errors = new ArrayList<>(); try { - if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getBackends()), - "At least 1 backend must be defined.")) - { - qInstance.getBackends().forEach((backendName, backend) -> - { - assertCondition(errors, Objects.equals(backendName, backend.getName()), - "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + "."); - }); - } - - ///////////////////////// - // validate the tables // - ///////////////////////// - if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getTables()), - "At least 1 table must be defined.")) - { - qInstance.getTables().forEach((tableName, table) -> - { - assertCondition(errors, Objects.equals(tableName, table.getName()), - "Inconsistent naming for table: " + tableName + "/" + table.getName() + "."); - - //////////////////////////////////////// - // validate the backend for the table // - //////////////////////////////////////// - if(assertCondition(errors, StringUtils.hasContent(table.getBackendName()), - "Missing backend name for table " + tableName + ".")) - { - if(CollectionUtils.nullSafeHasContents(qInstance.getBackends())) - { - assertCondition(errors, qInstance.getBackendForTable(tableName) != null, - "Unrecognized backend " + table.getBackendName() + " for table " + tableName + "."); - } - } - - ////////////////////////////////// - // validate fields in the table // - ////////////////////////////////// - if(assertCondition(errors, CollectionUtils.nullSafeHasContents(table.getFields()), - "At least 1 field must be defined in table " + tableName + ".")) - { - table.getFields().forEach((fieldName, field) -> - { - assertCondition(errors, Objects.equals(fieldName, field.getName()), - "Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + "."); - - if(field.getPossibleValueSourceName() != null) - { - assertCondition(errors, qInstance.getPossibleValueSource(field.getPossibleValueSourceName()) != null, - "Unrecognized possibleValueSourceName " + field.getPossibleValueSourceName() + " in table " + tableName + " for field " + fieldName + "."); - } - }); - } - }); - } + validateBackends(qInstance, errors); + validateTables(qInstance, errors); + validateProcesses(qInstance, errors); + validateApps(qInstance, errors); } catch(Exception e) { @@ -149,6 +104,181 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ + private void validateBackends(QInstance qInstance, List errors) + { + if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getBackends()), "At least 1 backend must be defined.")) + { + qInstance.getBackends().forEach((backendName, backend) -> + { + assertCondition(errors, Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + "."); + }); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void validateTables(QInstance qInstance, List errors) + { + if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getTables()), + "At least 1 table must be defined.")) + { + qInstance.getTables().forEach((tableName, table) -> + { + assertCondition(errors, Objects.equals(tableName, table.getName()), "Inconsistent naming for table: " + tableName + "/" + table.getName() + "."); + + validateAppChildHasValidParentAppName(qInstance, errors, table); + + //////////////////////////////////////// + // validate the backend for the table // + //////////////////////////////////////// + if(assertCondition(errors, StringUtils.hasContent(table.getBackendName()), + "Missing backend name for table " + tableName + ".")) + { + if(CollectionUtils.nullSafeHasContents(qInstance.getBackends())) + { + assertCondition(errors, qInstance.getBackendForTable(tableName) != null, "Unrecognized backend " + table.getBackendName() + " for table " + tableName + "."); + } + } + + ////////////////////////////////// + // validate fields in the table // + ////////////////////////////////// + if(assertCondition(errors, CollectionUtils.nullSafeHasContents(table.getFields()), "At least 1 field must be defined in table " + tableName + ".")) + { + table.getFields().forEach((fieldName, field) -> + { + assertCondition(errors, Objects.equals(fieldName, field.getName()), + "Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + "."); + + if(field.getPossibleValueSourceName() != null) + { + assertCondition(errors, qInstance.getPossibleValueSource(field.getPossibleValueSourceName()) != null, + "Unrecognized possibleValueSourceName " + field.getPossibleValueSourceName() + " in table " + tableName + " for field " + fieldName + "."); + } + }); + } + }); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void validateProcesses(QInstance qInstance, List errors) + { + if(!CollectionUtils.nullSafeIsEmpty(qInstance.getProcesses())) + { + qInstance.getProcesses().forEach((processName, process) -> + { + assertCondition(errors, Objects.equals(processName, process.getName()), "Inconsistent naming for process: " + processName + "/" + process.getName() + "."); + + validateAppChildHasValidParentAppName(qInstance, errors, process); + + ///////////////////////////////////////////// + // validate the table name for the process // + ///////////////////////////////////////////// + if(process.getTableName() != null) + { + assertCondition(errors, qInstance.getTable(process.getTableName()) != null, "Unrecognized table " + process.getTableName() + " for process " + processName + "."); + } + + /////////////////////////////////// + // validate steps in the process // + /////////////////////////////////// + if(assertCondition(errors, CollectionUtils.nullSafeHasContents(process.getStepList()), "At least 1 step must be defined in process " + processName + ".")) + { + int index = 0; + for(QStepMetaData step : process.getStepList()) + { + assertCondition(errors, StringUtils.hasContent(step.getName()), "Missing name for a step at index " + index + " in process " + processName); + index++; + } + } + }); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void validateApps(QInstance qInstance, List errors) + { + if(!CollectionUtils.nullSafeIsEmpty(qInstance.getApps())) + { + qInstance.getApps().forEach((appName, app) -> + { + assertCondition(errors, Objects.equals(appName, app.getName()), "Inconsistent naming for app: " + appName + "/" + app.getName() + "."); + + validateAppChildHasValidParentAppName(qInstance, errors, app); + + Set appsVisited = new HashSet<>(); + visitAppCheckingForCycles(app, appsVisited, errors); + + if(app.getChildren() != null) + { + Set childNames = new HashSet<>(); + for(QAppChildMetaData child : app.getChildren()) + { + assertCondition(errors, Objects.equals(appName, child.getParentAppName()), "Child " + child.getName() + " of app " + appName + " does not have its parent app properly set."); + assertCondition(errors, !childNames.contains(child.getName()), "App " + appName + " contains more than one child named " + child.getName()); + childNames.add(child.getName()); + } + } + }); + } + } + + + + /******************************************************************************* + ** Check if an app's child list can recursively be traversed without finding a + ** duplicate, which would indicate a cycle (e.g., an error) + *******************************************************************************/ + private void visitAppCheckingForCycles(QAppMetaData app, Set appsVisited, List errors) + { + if(assertCondition(errors, !appsVisited.contains(app.getName()), "Circular app reference detected, involving " + app.getName())) + { + appsVisited.add(app.getName()); + if(app.getChildren() != null) + { + for(QAppChildMetaData child : app.getChildren()) + { + if(child instanceof QAppMetaData childApp) + { + visitAppCheckingForCycles(childApp, appsVisited, errors); + } + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void validateAppChildHasValidParentAppName(QInstance qInstance, List errors, QAppChildMetaData appChild) + { + if(appChild.getParentAppName() != null) + { + assertCondition(errors, qInstance.getApp(appChild.getParentAppName()) != null, "Unrecognized parent app " + appChild.getParentAppName() + " for " + appChild.getName() + "."); + } + } + + + + /******************************************************************************* + ** For the given input condition, if it's true, then we're all good (and return true). + ** But if it's false, add the provided message to the list of errors (and return false, + ** e.g., in case you need to stop evaluating rules to avoid exceptions). + *******************************************************************************/ private boolean assertCondition(List errors, boolean condition, String message) { if(!condition) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java index 552cba50..db855917 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java @@ -87,7 +87,7 @@ public abstract class AbstractActionInput catch(QInstanceValidationException e) { LOG.warn(e); - throw (new IllegalArgumentException("QInstance failed validation" + e.getMessage())); + throw (new IllegalArgumentException("QInstance failed validation" + e.getMessage(), e)); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java index 2685bcfa..f47cc424 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java @@ -22,8 +22,10 @@ package com.kingsrook.qqq.backend.core.model.actions.metadata; +import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData; @@ -36,6 +38,9 @@ public class MetaDataOutput extends AbstractActionOutput { private Map tables; private Map processes; + private Map apps; + + private List appTree; @@ -80,4 +85,49 @@ public class MetaDataOutput extends AbstractActionOutput { this.processes = processes; } + + + + + /******************************************************************************* + ** Getter for appTree + ** + *******************************************************************************/ + public List getAppTree() + { + return appTree; + } + + + + /******************************************************************************* + ** Setter for appTree + ** + *******************************************************************************/ + public void setAppTree(List appTree) + { + this.appTree = appTree; + } + + + + /******************************************************************************* + ** Getter for apps + ** + *******************************************************************************/ + public Map getApps() + { + return apps; + } + + + + /******************************************************************************* + ** Setter for apps + ** + *******************************************************************************/ + public void setApps(Map apps) + { + this.apps = apps; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index 3f3a7f1f..2ef044ff 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -29,7 +29,9 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -69,6 +71,7 @@ public class QRecord implements Serializable } + /******************************************************************************* ** *******************************************************************************/ @@ -105,6 +108,16 @@ public class QRecord implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + public void setValue(QFieldMetaData field, Serializable value) + { + values.put(field.getName(), value); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -126,6 +139,16 @@ public class QRecord implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + public void setDisplayValue(QFieldMetaData field, Serializable rawValue) + { + displayValues.put(field.getName(), QValueFormatter.formatValue(field, rawValue)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -137,6 +160,17 @@ public class QRecord implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + public QRecord withDisplayValue(QFieldMetaData field, Serializable rawValue) + { + setDisplayValue(field, rawValue); + return (this); + } + + + /******************************************************************************* ** Getter for tableName ** @@ -355,6 +389,7 @@ public class QRecord implements Serializable } + /******************************************************************************* ** Getter for errors ** @@ -399,6 +434,7 @@ public class QRecord implements Serializable } + /******************************************************************************* ** Convert this record to an QRecordEntity *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 620f9f65..6f4c5230 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -24,10 +24,12 @@ package com.kingsrook.qqq.backend.core.model.metadata; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.instances.QInstanceValidationKey; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; @@ -49,9 +51,13 @@ public class QInstance private QAuthenticationMetaData authentication = null; - private Map tables = new HashMap<>(); - private Map> possibleValueSources = new HashMap<>(); - private Map processes = new HashMap<>(); + //////////////////////////////////////////////////////////////////////////////////////////// + // Important to use LinkedHashmap here, to preserve the order in which entries are added. // + //////////////////////////////////////////////////////////////////////////////////////////// + private Map tables = new LinkedHashMap<>(); + private Map> possibleValueSources = new LinkedHashMap<>(); + private Map processes = new LinkedHashMap<>(); + private Map apps = new LinkedHashMap<>(); // todo - lock down the object (no more changes allowed) after it's been validated? @@ -171,6 +177,11 @@ public class QInstance *******************************************************************************/ public QTableMetaData getTable(String name) { + if(this.tables == null) + { + return (null); + } + return (this.tables.get(name)); } @@ -260,6 +271,40 @@ public class QInstance + /******************************************************************************* + ** + *******************************************************************************/ + public void addApp(QAppMetaData app) + { + this.addApp(app.getName(), app); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addApp(String name, QAppMetaData app) + { + if(this.apps.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second app with name: " + name)); + } + this.apps.put(name, app); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QAppMetaData getApp(String name) + { + return (this.apps.get(name)); + } + + + /******************************************************************************* ** Getter for backends ** @@ -348,6 +393,28 @@ public class QInstance + /******************************************************************************* + ** Getter for apps + ** + *******************************************************************************/ + public Map getApps() + { + return apps; + } + + + + /******************************************************************************* + ** Setter for apps + ** + *******************************************************************************/ + public void setApps(Map apps) + { + this.apps = apps; + } + + + /******************************************************************************* ** Getter for hasBeenValidated ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DisplayFormat.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DisplayFormat.java new file mode 100644 index 00000000..4c884ed0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DisplayFormat.java @@ -0,0 +1,40 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.fields; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface DisplayFormat +{ + String DEFAULT = "%s"; + String STRING = "%s"; + String COMMAS = "%,d"; + String DECIMAL1_COMMAS = "%,.1f"; + String DECIMAL2_COMMAS = "%,.2f"; + String DECIMAL3_COMMAS = "%,.3f"; + String DECIMAL1 = "%.1f"; + String DECIMAL2 = "%.2f"; + String DECIMAL3 = "%.3f"; + String CURRENCY = "$%,.2f"; +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java index c5fb9e72..08fd34fa 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java @@ -47,6 +47,7 @@ public class QFieldMetaData // propose doing that in a secondary field, e.g., "onlyEditableOn=insert|update" // /////////////////////////////////////////////////////////////////////////////////// + private String displayFormat = "%s"; private Serializable defaultValue; private String possibleValueSourceName; @@ -354,4 +355,36 @@ public class QFieldMetaData return (this); } + + /******************************************************************************* + ** Getter for displayFormat + ** + *******************************************************************************/ + public String getDisplayFormat() + { + return displayFormat; + } + + + + /******************************************************************************* + ** Setter for displayFormat + ** + *******************************************************************************/ + public void setDisplayFormat(String displayFormat) + { + this.displayFormat = displayFormat; + } + + + /******************************************************************************* + ** Fluent setter for displayFormat + ** + *******************************************************************************/ + public QFieldMetaData withDisplayFormat(String displayFormat) + { + this.displayFormat = displayFormat; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java index 324e8991..79bc824a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java @@ -61,6 +61,10 @@ public enum QFieldType { return (INTEGER); } + if(c.equals(Boolean.class)) + { + return (BOOLEAN); + } if(c.equals(BigDecimal.class)) { return (DECIMAL); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNodeType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNodeType.java new file mode 100644 index 00000000..09c954f4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNodeType.java @@ -0,0 +1,33 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.frontend; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum AppTreeNodeType +{ + TABLE, + PROCESS, + APP +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java new file mode 100644 index 00000000..85d9d885 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java @@ -0,0 +1,164 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.frontend; + + +import java.util.ArrayList; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + * Version of QAppMetaData that's meant for transmitting to a frontend. + * + *******************************************************************************/ +@JsonInclude(Include.NON_NULL) +public class QFrontendAppMetaData +{ + private AppTreeNodeType type; + + private String name; + private String label; + + private List children = new ArrayList<>(); + + private String iconName; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QFrontendAppMetaData(QAppChildMetaData appChildMetaData) + { + this.name = appChildMetaData.getName(); + this.label = appChildMetaData.getLabel(); + + if(appChildMetaData.getClass().equals(QTableMetaData.class)) + { + this.type = AppTreeNodeType.TABLE; + } + else if(appChildMetaData.getClass().equals(QProcessMetaData.class)) + { + this.type = AppTreeNodeType.PROCESS; + } + else if(appChildMetaData.getClass().equals(QAppMetaData.class)) + { + this.type = AppTreeNodeType.APP; + } + else + { + throw (new IllegalStateException("Unrecognized class for app child meta data: " + appChildMetaData.getClass())); + } + + if(appChildMetaData.getIcon() != null && StringUtils.hasContent(appChildMetaData.getIcon().getName())) + { + this.iconName = appChildMetaData.getIcon().getName(); + } + } + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public AppTreeNodeType getType() + { + return type; + } + + + + /******************************************************************************* + ** Getter for children + ** + *******************************************************************************/ + public List getChildren() + { + return children; + } + + + + /******************************************************************************* + ** Getter for iconName + ** + *******************************************************************************/ + public String getIconName() + { + return iconName; + } + + + + /******************************************************************************* + ** Setter for iconName + ** + *******************************************************************************/ + public void setIconName(String iconName) + { + this.iconName = iconName; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addChild(QFrontendAppMetaData qFrontendAppMetaData) + { + if(children == null) + { + children = new ArrayList<>(); + } + children.add(qFrontendAppMetaData); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppChildMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppChildMetaData.java new file mode 100644 index 00000000..85db26ef --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppChildMetaData.java @@ -0,0 +1,58 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.layout; + + +/******************************************************************************* + ** Interface shared by meta-data objects which can be placed into an App. + *******************************************************************************/ +public interface QAppChildMetaData +{ + /******************************************************************************* + ** + *******************************************************************************/ + void setParentAppName(String parentAppName); + + /******************************************************************************* + ** + *******************************************************************************/ + String getParentAppName(); + + /******************************************************************************* + ** + *******************************************************************************/ + String getName(); + + /******************************************************************************* + ** + *******************************************************************************/ + String getLabel(); + + /******************************************************************************* + ** + *******************************************************************************/ + default QIcon getIcon() + { + return (null); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java new file mode 100644 index 00000000..22681a47 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java @@ -0,0 +1,237 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.layout; + + +import java.util.ArrayList; +import java.util.List; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QAppMetaData implements QAppChildMetaData +{ + private String name; + private String label; + + private List children; + + private String parentAppName; + private QIcon icon; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QAppMetaData() + { + } + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Setter for name + ** + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public QAppMetaData withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** Setter for label + ** + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + ** + *******************************************************************************/ + public QAppMetaData withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for children + ** + *******************************************************************************/ + public List getChildren() + { + return children; + } + + + + /******************************************************************************* + ** Setter for children + ** + *******************************************************************************/ + public void setChildren(List children) + { + this.children = children; + } + + + + /******************************************************************************* + ** Add a child to this app. + ** + *******************************************************************************/ + public void addChild(QAppChildMetaData child) + { + if(this.children == null) + { + this.children = new ArrayList<>(); + } + this.children.add(child); + child.setParentAppName(this.getName()); + } + + + + /******************************************************************************* + ** Fluently add a child to this app. + ** + *******************************************************************************/ + public QAppMetaData withChild(QAppChildMetaData child) + { + addChild(child); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for children + ** + *******************************************************************************/ + public QAppMetaData withChildren(List children) + { + this.children = children; + return (this); + } + + + + /******************************************************************************* + ** Getter for parentAppName + ** + *******************************************************************************/ + @Override + public String getParentAppName() + { + return parentAppName; + } + + + + /******************************************************************************* + ** Setter for parentAppName + ** + *******************************************************************************/ + @Override + public void setParentAppName(String parentAppName) + { + this.parentAppName = parentAppName; + } + + + /******************************************************************************* + ** Getter for icon + ** + *******************************************************************************/ + public QIcon getIcon() + { + return icon; + } + + + + /******************************************************************************* + ** Setter for icon + ** + *******************************************************************************/ + public void setIcon(QIcon icon) + { + this.icon = icon; + } + + + /******************************************************************************* + ** Fluent setter for icon + ** + *******************************************************************************/ + public QAppMetaData withIcon(QIcon icon) + { + this.icon = icon; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QIcon.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QIcon.java new file mode 100644 index 00000000..92e374c6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QIcon.java @@ -0,0 +1,91 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.layout; + + +/******************************************************************************* + ** Icon to show associated with an App, Table, Process, etc. + ** + ** Currently, name here must be a reference from https://fonts.google.com/icons + ** e.g., local_shipping for https://fonts.google.com/icons?selected=Material+Symbols+Outlined:local_shipping + ** + ** Future may allow something like a "namespace", and/or multiple icons for + ** use in different frontends, etc. + *******************************************************************************/ +public class QIcon +{ + private String name; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QIcon() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QIcon(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Setter for name + ** + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public QIcon withName(String name) + { + this.name = name; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index 213eabd2..a63f66ba 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -26,13 +26,15 @@ import java.util.ArrayList; import java.util.List; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; /******************************************************************************* ** Meta-Data to define a process in a QQQ instance. ** *******************************************************************************/ -public class QProcessMetaData +public class QProcessMetaData implements QAppChildMetaData { private String name; private String label; @@ -41,6 +43,8 @@ public class QProcessMetaData private List stepList; + private String parentAppName; + /******************************************************************************* @@ -293,4 +297,28 @@ public class QProcessMetaData return (this); } + + + /******************************************************************************* + ** Getter for parentAppName + ** + *******************************************************************************/ + @Override + public String getParentAppName() + { + return parentAppName; + } + + + + /******************************************************************************* + ** Setter for parentAppName + ** + *******************************************************************************/ + @Override + public void setParentAppName(String parentAppName) + { + this.parentAppName = parentAppName; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index aed8f51c..506ca3d4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -30,13 +30,15 @@ import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; /******************************************************************************* ** Meta-Data to define a table in a QQQ instance. ** *******************************************************************************/ -public class QTableMetaData implements Serializable +public class QTableMetaData implements QAppChildMetaData, Serializable { private String name; private String label; @@ -59,6 +61,8 @@ public class QTableMetaData implements Serializable private Map customizers; + private String parentAppName; + private QIcon icon; /******************************************************************************* @@ -434,4 +438,72 @@ public class QTableMetaData implements Serializable return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData withInferredFieldBackendNames() + { + // todo not commit QInstanceEnricher.setInferredFieldBackendNames(this); + return (this); + } + + + + /******************************************************************************* + ** Getter for parentAppName + ** + *******************************************************************************/ + @Override + public String getParentAppName() + { + return parentAppName; + } + + + + /******************************************************************************* + ** Setter for parentAppName + ** + *******************************************************************************/ + @Override + public void setParentAppName(String parentAppName) + { + this.parentAppName = parentAppName; + } + + + + /******************************************************************************* + ** Getter for icon + ** + *******************************************************************************/ + public QIcon getIcon() + { + return icon; + } + + + + /******************************************************************************* + ** Setter for icon + ** + *******************************************************************************/ + public void setIcon(QIcon icon) + { + this.icon = icon; + } + + + /******************************************************************************* + ** Fluent setter for icon + ** + *******************************************************************************/ + public QTableMetaData withIcon(QIcon icon) + { + this.icon = icon; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java index ff3761f7..ec5c3365 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java @@ -65,6 +65,7 @@ public class MockQueryAction implements QueryInterface { Serializable value = field.equals("id") ? (i + 1) : getValue(table, field); record.setValue(field, value); + record.setDisplayValue(table.getField(field), value); } queryOutput.addRecord(record); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java index f41b7012..c86ef97e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java @@ -22,12 +22,20 @@ package com.kingsrook.qqq.backend.core.actions.metadata; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; /******************************************************************************* @@ -47,9 +55,66 @@ class MetaDataActionTest request.setSession(TestUtils.getMockSession()); MetaDataOutput result = new MetaDataAction().execute(request); assertNotNull(result); + + /////////////////////////////////// + // assert against the tables map // + /////////////////////////////////// assertNotNull(result.getTables()); assertNotNull(result.getTables().get("person")); assertEquals("person", result.getTables().get("person").getName()); assertEquals("Person", result.getTables().get("person").getLabel()); + + ////////////////////////////////////// + // assert against the processes map // + ////////////////////////////////////// + assertNotNull(result.getProcesses().get("greet")); + assertNotNull(result.getProcesses().get("greetInteractive")); + assertNotNull(result.getProcesses().get("etl.basic")); + assertNotNull(result.getProcesses().get("person.bulkInsert")); + assertNotNull(result.getProcesses().get("person.bulkEdit")); + assertNotNull(result.getProcesses().get("person.bulkDelete")); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // assert against the apps map - which is appName to app - but not fully hierarchical - that's appTree // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + Map apps = result.getApps(); + assertNotNull(apps.get(TestUtils.APP_NAME_GREETINGS)); + assertNotNull(apps.get(TestUtils.APP_NAME_PEOPLE)); + assertNotNull(apps.get(TestUtils.APP_NAME_MISCELLANEOUS)); + + QFrontendAppMetaData peopleApp = apps.get(TestUtils.APP_NAME_PEOPLE); + assertThat(peopleApp.getChildren()).isNotEmpty(); + Optional greetingsAppUnderPeopleFromMapOptional = peopleApp.getChildren().stream() + .filter(e -> e.getName().equals(TestUtils.APP_NAME_GREETINGS)).findFirst(); + assertThat(greetingsAppUnderPeopleFromMapOptional).isPresent(); + + ////////////////////////////////////////////////////////////////////////////// + // we want to show that in the appMap (e.g., "apps"), that the apps are not // + // hierarchical - that is - that a sub-app doesn't list ITS children here. // + ////////////////////////////////////////////////////////////////////////////// + assertThat(greetingsAppUnderPeopleFromMapOptional.get().getChildren()).isNullOrEmpty(); + + /////////////////////////////////////////////// + // assert against the hierarchical apps tree // + /////////////////////////////////////////////// + List appTree = result.getAppTree(); + Set appNamesInTopOfTree = appTree.stream().map(QFrontendAppMetaData::getName).collect(Collectors.toSet()); + assertThat(appNamesInTopOfTree).contains(TestUtils.APP_NAME_PEOPLE); + assertThat(appNamesInTopOfTree).contains(TestUtils.APP_NAME_MISCELLANEOUS); + assertThat(appNamesInTopOfTree).doesNotContain(TestUtils.APP_NAME_GREETINGS); + + Optional peopleAppOptional = appTree.stream() + .filter(e -> e.getName().equals(TestUtils.APP_NAME_PEOPLE)).findFirst(); + assertThat(peopleAppOptional).isPresent(); + assertThat(peopleAppOptional.get().getChildren()).isNotEmpty(); + + Optional greetingsAppUnderPeopleFromTree = peopleAppOptional.get().getChildren().stream() + .filter(e -> e.getName().equals(TestUtils.APP_NAME_GREETINGS)).findFirst(); + assertThat(greetingsAppUnderPeopleFromTree).isPresent(); + + ///////////////////////////////////////////////////////////////////////////////// + // but here, when this app comes from the tree, then it DOES have its children // + ///////////////////////////////////////////////////////////////////////////////// + assertThat(greetingsAppUnderPeopleFromTree.get().getChildren()).isNotEmpty(); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java index 3917f7e7..326e0742 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java @@ -25,8 +25,10 @@ package com.kingsrook.qqq.backend.core.actions.tables; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -50,5 +52,13 @@ class QueryActionTest request.setTableName("person"); QueryOutput result = new QueryAction().execute(request); assertNotNull(result); + + assertThat(result.getRecords()).isNotEmpty(); + for(QRecord record : result.getRecords()) + { + assertThat(record.getValues()).isNotEmpty(); + assertThat(record.getDisplayValues()).isNotEmpty(); + assertThat(record.getErrors()).isEmpty(); + } } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index c2bc70e9..743f43ef 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -22,13 +22,17 @@ package com.kingsrook.qqq.backend.core.instances; +import java.util.Collections; import java.util.HashMap; +import java.util.function.Consumer; import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -51,6 +55,21 @@ class QInstanceValidatorTest + /******************************************************************************* + ** make sure we don't re-validate if already validated + ** + *******************************************************************************/ + @Test + public void test_doNotReValidate() throws QInstanceValidationException + { + QInstance qInstance = TestUtils.defineInstance(); + qInstance.setHasBeenValidated(new QInstanceValidationKey()); + qInstance.setBackends(null); + new QInstanceValidator().validate(qInstance); + } + + + /******************************************************************************* ** Test an instance with null backends - should throw. ** @@ -58,17 +77,8 @@ class QInstanceValidatorTest @Test public void test_validateNullBackends() { - try - { - QInstance qInstance = TestUtils.defineInstance(); - qInstance.setBackends(null); - new QInstanceValidator().validate(qInstance); - fail("Should have thrown validationException"); - } - catch(QInstanceValidationException e) - { - assertReason("At least 1 backend must be defined", e); - } + assertValidationFailureReasons((qInstance) -> qInstance.setBackends(null), + "At least 1 backend must be defined"); } @@ -80,17 +90,8 @@ class QInstanceValidatorTest @Test public void test_validateEmptyBackends() { - try - { - QInstance qInstance = TestUtils.defineInstance(); - qInstance.setBackends(new HashMap<>()); - new QInstanceValidator().validate(qInstance); - fail("Should have thrown validationException"); - } - catch(QInstanceValidationException e) - { - assertReason("At least 1 backend must be defined", e); - } + assertValidationFailureReasons((qInstance) -> qInstance.setBackends(new HashMap<>()), + "At least 1 backend must be defined"); } @@ -102,17 +103,12 @@ class QInstanceValidatorTest @Test public void test_validateNullTables() { - try - { - QInstance qInstance = TestUtils.defineInstance(); - qInstance.setTables(null); - new QInstanceValidator().validate(qInstance); - fail("Should have thrown validationException"); - } - catch(QInstanceValidationException e) - { - assertReason("At least 1 table must be defined", e); - } + assertValidationFailureReasons((qInstance) -> + { + qInstance.setTables(null); + qInstance.setProcesses(null); + }, + "At least 1 table must be defined"); } @@ -124,17 +120,12 @@ class QInstanceValidatorTest @Test public void test_validateEmptyTables() { - try - { - QInstance qInstance = TestUtils.defineInstance(); - qInstance.setTables(new HashMap<>()); - new QInstanceValidator().validate(qInstance); - fail("Should have thrown validationException"); - } - catch(QInstanceValidationException e) - { - assertReason("At least 1 table must be defined", e); - } + assertValidationFailureReasons((qInstance) -> + { + qInstance.setTables(new HashMap<>()); + qInstance.setProcesses(new HashMap<>()); + }, + "At least 1 table must be defined"); } @@ -147,19 +138,15 @@ class QInstanceValidatorTest @Test public void test_validateInconsistentNames() { - try - { - QInstance qInstance = TestUtils.defineInstance(); - qInstance.getTable("person").setName("notPerson"); - qInstance.getBackend("default").setName("notDefault"); - new QInstanceValidator().validate(qInstance); - fail("Should have thrown validationException"); - } - catch(QInstanceValidationException e) - { - assertReason("Inconsistent naming for table", e); - assertReason("Inconsistent naming for backend", e); - } + assertValidationFailureReasonsAllowingExtraReasons((qInstance) -> + { + qInstance.getTable("person").setName("notPerson"); + qInstance.getBackend("default").setName("notDefault"); + qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setName("notGreetPeople"); + }, + "Inconsistent naming for table", + "Inconsistent naming for backend", + "Inconsistent naming for process"); } @@ -171,17 +158,8 @@ class QInstanceValidatorTest @Test public void test_validateTableWithoutBackend() { - try - { - QInstance qInstance = TestUtils.defineInstance(); - qInstance.getTable("person").setBackendName(null); - new QInstanceValidator().validate(qInstance); - fail("Should have thrown validationException"); - } - catch(QInstanceValidationException e) - { - assertReason("Missing backend name for table", e); - } + assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setBackendName(null), + "Missing backend name for table"); } @@ -193,17 +171,53 @@ class QInstanceValidatorTest @Test public void test_validateTableWithMissingBackend() { - try - { - QInstance qInstance = TestUtils.defineInstance(); - qInstance.getTable("person").setBackendName("notARealBackend"); - new QInstanceValidator().validate(qInstance); - fail("Should have thrown validationException"); - } - catch(QInstanceValidationException e) - { - assertReason("Unrecognized backend", e); - } + assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setBackendName("notARealBackend"), + "Unrecognized backend"); + } + + + + /******************************************************************************* + ** Test that if a process specifies a table that doesn't exist, that it fails. + ** + *******************************************************************************/ + @Test + public void test_validateProcessWithMissingTable() + { + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setTableName("notATableName"), + "Unrecognized table"); + } + + + + /******************************************************************************* + ** Test that a process with no steps fails + ** + *******************************************************************************/ + @Test + public void test_validateProcessWithNoSteps() + { + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setStepList(Collections.emptyList()), + "At least 1 step"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setStepList(null), + "At least 1 step"); + } + + + + /******************************************************************************* + ** Test that a process step with an empty string name fails + ** + *******************************************************************************/ + @Test + public void test_validateProcessStepWithEmptyName() + { + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).getStepList().get(0).setName(""), + "Missing name for a step"); + + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE).getStepList().get(1).setName(null), + "Missing name for a step"); } @@ -215,29 +229,11 @@ class QInstanceValidatorTest @Test public void test_validateTableWithNoFields() { - try - { - QInstance qInstance = TestUtils.defineInstance(); - qInstance.getTable("person").setFields(null); - new QInstanceValidator().validate(qInstance); - fail("Should have thrown validationException"); - } - catch(QInstanceValidationException e) - { - assertReason("At least 1 field", e); - } + assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setFields(null), + "At least 1 field"); - try - { - QInstance qInstance = TestUtils.defineInstance(); - qInstance.getTable("person").setFields(new HashMap<>()); - new QInstanceValidator().validate(qInstance); - fail("Should have thrown validationException"); - } - catch(QInstanceValidationException e) - { - assertReason("At least 1 field", e); - } + assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").setFields(new HashMap<>()), + "At least 1 field"); } @@ -248,17 +244,92 @@ class QInstanceValidatorTest *******************************************************************************/ @Test public void test_validateFieldWithMissingPossibleValueSource() + { + assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").getField("homeState").setPossibleValueSourceName("not a real possible value source"), + "Unrecognized possibleValueSourceName"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testChildrenWithBadParentAppName() + { + String[] reasons = new String[] { "Unrecognized parent app", "does not have its parent app properly set" }; + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON).setParentAppName("notAnApp"), reasons); + assertValidationFailureReasons((qInstance) -> qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setParentAppName("notAnApp"), reasons); + assertValidationFailureReasons((qInstance) -> qInstance.getApp(TestUtils.APP_NAME_GREETINGS).setParentAppName("notAnApp"), reasons); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testAppCircularReferences() + { + assertValidationFailureReasonsAllowingExtraReasons((qInstance) -> + { + QAppMetaData miscApp = qInstance.getApp(TestUtils.APP_NAME_MISCELLANEOUS); + QAppMetaData greetingsApp = qInstance.getApp(TestUtils.APP_NAME_GREETINGS); + + miscApp.withChild(greetingsApp); + greetingsApp.withChild(miscApp); + }, "Circular app reference"); + } + + + + /******************************************************************************* + ** Run a little setup code on a qInstance; then validate it, and assert that it + ** failed validation with reasons that match the supplied vararg-reasons (but allow + ** more reasons - e.g., helpful when one thing we're testing causes other errors). + *******************************************************************************/ + private void assertValidationFailureReasonsAllowingExtraReasons(Consumer setup, String... reasons) + { + assertValidationFailureReasons(setup, true, reasons); + } + + + + /******************************************************************************* + ** Run a little setup code on a qInstance; then validate it, and assert that it + ** failed validation with reasons that match the supplied vararg-reasons (and + ** require that exact # of reasons). + *******************************************************************************/ + private void assertValidationFailureReasons(Consumer setup, String... reasons) + { + assertValidationFailureReasons(setup, false, reasons); + } + + + + /******************************************************************************* + ** Implementation for the overloads of this name. + *******************************************************************************/ + private void assertValidationFailureReasons(Consumer setup, boolean allowExtraReasons, String... reasons) { try { QInstance qInstance = TestUtils.defineInstance(); - qInstance.getTable("person").getField("homeState").setPossibleValueSourceName("not a real possible value source"); + setup.accept(qInstance); new QInstanceValidator().validate(qInstance); fail("Should have thrown validationException"); } catch(QInstanceValidationException e) { - assertReason("Unrecognized possibleValueSourceName", e); + if(!allowExtraReasons) + { + assertEquals(reasons.length, e.getReasons().size(), "Expected number of validation failure reasons\nExpected: " + String.join(",", reasons) + "\nActual: " + e.getReasons()); + } + + for(String reason : reasons) + { + assertReason(reason, e); + } } } @@ -271,7 +342,9 @@ class QInstanceValidatorTest *******************************************************************************/ private void assertReason(String reason, QInstanceValidationException e) { - assertNotNull(e.getReasons()); - assertTrue(e.getReasons().stream().anyMatch(s -> s.contains(reason))); + assertNotNull(e.getReasons(), "Expected there to be a reason for the failure (but there was not)"); + assertThat(e.getReasons()) + .withFailMessage("Expected any of:\n%s\nTo match: [%s]", e.getReasons(), reason) + .anyMatch(s -> s.contains(reason)); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 6bae6380..61365412 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; @@ -62,9 +63,19 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicE *******************************************************************************/ public class TestUtils { - public static String DEFAULT_BACKEND_NAME = "default"; - public static String PROCESS_NAME_GREET_PEOPLE = "greet"; - public static String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive"; + public static final String DEFAULT_BACKEND_NAME = "default"; + + public static final String APP_NAME_GREETINGS = "greetingsApp"; + public static final String APP_NAME_PEOPLE = "peopleApp"; + public static final String APP_NAME_MISCELLANEOUS = "miscellaneous"; + + public static final String TABLE_NAME_PERSON = "person"; + + public static final String PROCESS_NAME_GREET_PEOPLE = "greet"; + public static final String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive"; + public static final String PROCESS_NAME_ADD_TO_PEOPLES_AGE = "addToPeoplesAge"; + public static final String TABLE_NAME_PERSON_FILE = "personFile"; + public static final String TABLE_NAME_ID_AND_NAME_ONLY = "idAndNameOnly"; @@ -77,15 +88,20 @@ public class TestUtils QInstance qInstance = new QInstance(); qInstance.setAuthentication(defineAuthentication()); qInstance.addBackend(defineBackend()); + qInstance.addTable(defineTablePerson()); qInstance.addTable(definePersonFileTable()); qInstance.addTable(defineTableIdAndNameOnly()); + qInstance.addPossibleValueSource(defineStatesPossibleValueSource()); + qInstance.addProcess(defineProcessGreetPeople()); qInstance.addProcess(defineProcessGreetPeopleInteractive()); qInstance.addProcess(defineProcessAddToPeoplesAge()); qInstance.addProcess(new BasicETLProcess().defineProcessMetaData()); + defineApps(qInstance); + System.out.println(new QInstanceAdapter().qInstanceToJson(qInstance)); return (qInstance); @@ -93,6 +109,30 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + private static void defineApps(QInstance qInstance) + { + qInstance.addApp(new QAppMetaData() + .withName(APP_NAME_GREETINGS) + .withChild(qInstance.getProcess(PROCESS_NAME_GREET_PEOPLE)) + .withChild(qInstance.getProcess(PROCESS_NAME_GREET_PEOPLE_INTERACTIVE))); + + qInstance.addApp(new QAppMetaData() + .withName(APP_NAME_PEOPLE) + .withChild(qInstance.getTable(TABLE_NAME_PERSON)) + .withChild(qInstance.getTable(TABLE_NAME_PERSON_FILE)) + .withChild(qInstance.getApp(APP_NAME_GREETINGS))); + + qInstance.addApp(new QAppMetaData() + .withName(APP_NAME_MISCELLANEOUS) + .withChild(qInstance.getTable(TABLE_NAME_ID_AND_NAME_ONLY)) + .withChild(qInstance.getProcess(BasicETLProcess.PROCESS_NAME))); + } + + + /******************************************************************************* ** Define the "states" possible value source used in standard tests ** diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 2dddf8d6..06fc6880 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -46,10 +46,12 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter; import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.exceptions.QValueException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput; @@ -121,7 +123,7 @@ public class QJavalinImplementation /******************************************************************************* ** *******************************************************************************/ - public static void main(String[] args) + public static void main(String[] args) throws QInstanceValidationException { QInstance qInstance = new QInstance(); // todo - parse args to look up metaData and prime instance @@ -135,9 +137,10 @@ public class QJavalinImplementation /******************************************************************************* ** *******************************************************************************/ - public QJavalinImplementation(QInstance qInstance) + public QJavalinImplementation(QInstance qInstance) throws QInstanceValidationException { QJavalinImplementation.qInstance = qInstance; + new QInstanceValidator().validate(qInstance); } diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinTestBase.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinTestBase.java index 168174c2..571013fa 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinTestBase.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinTestBase.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.javalin; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -44,7 +45,7 @@ public class QJavalinTestBase ** *******************************************************************************/ @BeforeAll - public static void beforeAll() + public static void beforeAll() throws QInstanceValidationException { qJavalinImplementation = new QJavalinImplementation(TestUtils.defineInstance()); QJavalinProcessHandler.setAsyncStepTimeoutMillis(250); diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java index 434b4362..aa5b049b 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java @@ -22,6 +22,7 @@ package com.kingsrook.sampleapp; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.javalin.QJavalinImplementation; import io.javalin.Javalin; diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index a86b7ad4..87458a05 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -29,13 +29,13 @@ import com.kingsrook.qqq.backend.core.exceptions.QValueException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; -import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; @@ -59,7 +59,7 @@ import io.github.cdimascio.dotenv.Dotenv; *******************************************************************************/ public class SampleMetaDataProvider { - public static boolean USE_MYSQL = false; + public static boolean USE_MYSQL = true; public static final String RDBMS_BACKEND_NAME = "rdbms"; public static final String FILESYSTEM_BACKEND_NAME = "filesystem"; @@ -68,12 +68,20 @@ public class SampleMetaDataProvider // public static final String AUTH0_BASE_URL = "https://kingsrook.us.auth0.com/"; public static final String AUTH0_BASE_URL = "https://nutrifresh-one-development.us.auth0.com/"; + public static final String APP_NAME_GREETINGS = "greetingsApp"; + public static final String APP_NAME_PEOPLE = "peopleApp"; + public static final String APP_NAME_MISCELLANEOUS = "miscellaneous"; + public static final String PROCESS_NAME_GREET = "greet"; public static final String PROCESS_NAME_GREET_INTERACTIVE = "greetInteractive"; public static final String PROCESS_NAME_SIMPLE_SLEEP = "simpleSleep"; public static final String PROCESS_NAME_SIMPLE_THROW = "simpleThrow"; public static final String PROCESS_NAME_SLEEP_INTERACTIVE = "sleepInteractive"; + public static final String TABLE_NAME_PERSON = "person"; + public static final String TABLE_NAME_CARRIER = "carrier"; + public static final String TABLE_NAME_CITY = "city"; + public static final String STEP_NAME_SLEEPER = "sleeper"; public static final String STEP_NAME_THROWER = "thrower"; @@ -101,11 +109,39 @@ public class SampleMetaDataProvider qInstance.addProcess(defineProcessScreenThenSleep()); qInstance.addProcess(defineProcessSimpleThrow()); + defineApps(qInstance); + return (qInstance); } + /******************************************************************************* + ** + *******************************************************************************/ + private static void defineApps(QInstance qInstance) + { + qInstance.addApp(new QAppMetaData() + .withName(APP_NAME_GREETINGS) + .withChild(qInstance.getProcess(PROCESS_NAME_GREET)) + .withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE))); + + qInstance.addApp(new QAppMetaData() + .withName(APP_NAME_PEOPLE) + .withChild(qInstance.getTable(TABLE_NAME_PERSON)) + .withChild(qInstance.getTable(TABLE_NAME_CITY)) + .withChild(qInstance.getApp(APP_NAME_GREETINGS))); + + qInstance.addApp(new QAppMetaData() + .withName(APP_NAME_MISCELLANEOUS) + .withChild(qInstance.getTable(TABLE_NAME_CARRIER)) + .withChild(qInstance.getProcess(PROCESS_NAME_SIMPLE_SLEEP)) + .withChild(qInstance.getProcess(PROCESS_NAME_SLEEP_INTERACTIVE)) + .withChild(qInstance.getProcess(PROCESS_NAME_SIMPLE_THROW))); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -166,7 +202,7 @@ public class SampleMetaDataProvider public static QTableMetaData defineTableCarrier() { QTableMetaData table = new QTableMetaData(); - table.setName("carrier"); + table.setName(TABLE_NAME_CARRIER); table.setBackendName(RDBMS_BACKEND_NAME); table.setPrimaryKeyField("id"); @@ -194,7 +230,7 @@ public class SampleMetaDataProvider public static QTableMetaData defineTablePerson() { return new QTableMetaData() - .withName("person") + .withName(TABLE_NAME_PERSON) .withLabel("Person") .withBackendName(RDBMS_BACKEND_NAME) .withPrimaryKeyField("id") @@ -215,7 +251,7 @@ public class SampleMetaDataProvider public static QTableMetaData defineTableCityFile() { return new QTableMetaData() - .withName("city") + .withName(TABLE_NAME_CITY) .withLabel("Cities") .withIsHidden(true) .withBackendName(FILESYSTEM_BACKEND_NAME) @@ -240,7 +276,7 @@ public class SampleMetaDataProvider return new QProcessMetaData() .withName(PROCESS_NAME_GREET) .withLabel("Greet People") - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .withIsHidden(true) .addStep(new QBackendStepMetaData() .withName("prepare") @@ -249,14 +285,14 @@ public class SampleMetaDataProvider .withCodeType(QCodeType.JAVA) .withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context? .withInputData(new QFunctionInputMetaData() - .withRecordListMetaData(new QRecordListMetaData().withTableName("person")) + .withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON)) .withFieldList(List.of( new QFieldMetaData("greetingPrefix", QFieldType.STRING), new QFieldMetaData("greetingSuffix", QFieldType.STRING) ))) .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) @@ -272,9 +308,9 @@ public class SampleMetaDataProvider { return new QProcessMetaData() .withName(PROCESS_NAME_GREET_INTERACTIVE) - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) - .addStep(LoadInitialRecordsStep.defineMetaData("person")) + .addStep(LoadInitialRecordsStep.defineMetaData(TABLE_NAME_PERSON)) .addStep(new QFrontendStepMetaData() .withName("setup") @@ -289,14 +325,14 @@ public class SampleMetaDataProvider .withCodeType(QCodeType.JAVA) .withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context? .withInputData(new QFunctionInputMetaData() - .withRecordListMetaData(new QRecordListMetaData().withTableName("person")) + .withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON)) .withFieldList(List.of( new QFieldMetaData("greetingPrefix", QFieldType.STRING), new QFieldMetaData("greetingSuffix", QFieldType.STRING) ))) .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() - .withTableName("person") + .withTableName(TABLE_NAME_PERSON) .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) From 96dcfd2d54defbcb383268a991efbb4f306b80e4 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 4 Aug 2022 18:53:37 -0500 Subject: [PATCH 12/18] Add writing of .env file --- .circleci/config.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e7638e3..a244b41a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,6 +31,10 @@ commands: - restore_cache: keys: - v1-dependencies-{{ checksum "pom.xml" }} + - run: + name: Write .env + command: | + echo "RDBMS_PASSWORD=$RDBMS_PASSWORD" >> .env - run: name: Run Maven Verify command: | @@ -99,7 +103,7 @@ workflows: test_only: jobs: - mvn_test: - context: [ qqq-maven-registry-credentials, kingsrook-slack ] + context: [ qqq-maven-registry-credentials, kingsrook-slack, build-qqq-sample-app ] filters: branches: ignore: /dev/ @@ -109,7 +113,7 @@ workflows: deploy: jobs: - mvn_deploy: - context: [ qqq-maven-registry-credentials, kingsrook-slack ] + context: [ qqq-maven-registry-credentials, kingsrook-slack, build-qqq-sample-app ] filters: branches: only: /dev/ From 389e1adaeaeff96880c7df59de38c30bb5b5132b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 4 Aug 2022 19:03:36 -0500 Subject: [PATCH 13/18] Fixed path to qqq-sample-project/.env --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a244b41a..24447e0a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,7 @@ commands: - run: name: Write .env command: | - echo "RDBMS_PASSWORD=$RDBMS_PASSWORD" >> .env + echo "RDBMS_PASSWORD=$RDBMS_PASSWORD" >> qqq-sample-project/.env - run: name: Run Maven Verify command: | From 27fdefb6eef27ac20bbd44a53ee750411dc40cff Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 4 Aug 2022 19:08:08 -0500 Subject: [PATCH 14/18] Disabing this test in CI --- .../com/kingsrook/sampleapp/SampleMetaDataProviderTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleMetaDataProviderTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleMetaDataProviderTest.java index 1b145b57..6192bb38 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleMetaDataProviderTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleMetaDataProviderTest.java @@ -49,6 +49,8 @@ import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -57,6 +59,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* ** *******************************************************************************/ +@DisabledOnOs(OS.LINUX) // uses database; not available in CI at this time... class SampleMetaDataProviderTest { From 36d1a75ce6298779794613372e4a2ccf5c242e22 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 5 Aug 2022 07:59:15 -0500 Subject: [PATCH 15/18] Add -T4 (4 threads) arg to mvn commands --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 24447e0a..244b6d2b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,7 +38,7 @@ commands: - run: name: Run Maven Verify command: | - mvn -s .circleci/mvn-settings.xml verify + mvn -s .circleci/mvn-settings.xml -T4 verify - store_jacoco_site: module: qqq-backend-core - store_jacoco_site: @@ -73,7 +73,7 @@ commands: - run: name: Run Maven Jar Deploy command: | - mvn -s .circleci/mvn-settings.xml flatten:flatten jar:jar deploy:deploy + mvn -s .circleci/mvn-settings.xml -T4 flatten:flatten jar:jar deploy:deploy - save_cache: paths: - ~/.m2 From 68af23673895cfc9df2ab1df24cb83d7f9cb479f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 5 Aug 2022 08:09:02 -0500 Subject: [PATCH 16/18] Add option to retur stored records; add LocalTime --- .../core/model/data/QRecordEntityField.java | 6 +++ .../etl/basic/BasicETLLoadFunction.java | 19 +++++++- .../qqq/backend/core/utils/ValueUtils.java | 43 +++++++++++++++++++ .../backend/core/utils/ValueUtilsTest.java | 28 +++++++++++- 4 files changed, 93 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java index f977f244..510308c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java @@ -27,6 +27,7 @@ import java.lang.reflect.Method; import java.math.BigDecimal; import java.time.Instant; import java.time.LocalDate; +import java.time.LocalTime; import com.kingsrook.qqq.backend.core.exceptions.QValueException; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -159,6 +160,11 @@ public class QRecordEntityField { return (ValueUtils.getValueAsInstant(value)); } + + if(type.equals(LocalTime.class)) + { + return (ValueUtils.getValueAsLocalTime(value)); + } } catch(Exception e) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java index 997e377b..47f71df6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java @@ -44,6 +44,8 @@ public class BasicETLLoadFunction implements BackendStep { private static final Logger LOG = LogManager.getLogger(BasicETLLoadFunction.class); + private boolean returnStoredRecords = false; + /******************************************************************************* @@ -89,7 +91,11 @@ public class BasicETLLoadFunction implements BackendStep InsertAction insertAction = new InsertAction(); InsertOutput insertOutput = insertAction.execute(insertInput); - outputRecords.addAll(insertOutput.getRecords()); + + if(returnStoredRecords) + { + outputRecords.addAll(insertOutput.getRecords()); + } recordsInserted += insertOutput.getRecords().size(); } @@ -97,4 +103,15 @@ public class BasicETLLoadFunction implements BackendStep runBackendStepOutput.addValue(BasicETLProcess.FIELD_RECORD_COUNT, recordsInserted); } + + + /******************************************************************************* + ** Setter for returnStoredRecords + ** + *******************************************************************************/ + public void setReturnStoredRecords(boolean returnStoredRecords) + { + this.returnStoredRecords = returnStoredRecords; + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index d839a36b..a33abaa1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -22,10 +22,12 @@ package com.kingsrook.qqq.backend.core.utils; +import java.io.Serializable; import java.math.BigDecimal; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; @@ -415,4 +417,45 @@ public class ValueUtils throw (new QValueException("Value [" + value + "] could not be converted to a Instant.", e)); } } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Object getValueAsLocalTime(Serializable value) + { + try + { + if(value == null) + { + return (null); + } + else if(value instanceof LocalTime lt) + { + return (lt); + } + else if(value instanceof String s) + { + if(!StringUtils.hasContent(s)) + { + return (null); + } + + return LocalTime.parse(s); + } + else + { + throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to LocalTime.")); + } + } + catch(QValueException qve) + { + throw (qve); + } + catch(Exception e) + { + throw (new QValueException("Value [" + value + "] could not be converted to a LocalTime.", e)); + } + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java index abb4afbc..0ef501a7 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java @@ -27,14 +27,18 @@ import java.math.MathContext; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.Month; -import java.time.ZoneOffset; import java.util.Calendar; import java.util.GregorianCalendar; import com.kingsrook.qqq.backend.core.exceptions.QValueException; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* @@ -226,4 +230,24 @@ class ValueUtilsTest } + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetValueAsLocalTime() throws QValueException + { + assertNull(ValueUtils.getValueAsInstant(null)); + assertNull(ValueUtils.getValueAsInstant("")); + assertNull(ValueUtils.getValueAsInstant(" ")); + assertEquals(LocalTime.of(10, 42), ValueUtils.getValueAsLocalTime(LocalTime.of(10, 42))); + assertEquals(LocalTime.of(10, 42, 59), ValueUtils.getValueAsLocalTime(LocalTime.of(10, 42, 59))); + assertEquals(LocalTime.of(10, 42), ValueUtils.getValueAsLocalTime("10:42")); + assertEquals(LocalTime.of(10, 42, 59), ValueUtils.getValueAsLocalTime("10:42:59")); + + assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("a")); + assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("a,b")); + assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("1980/05/31")); + assertThat(assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant(new Object())).getMessage()).contains("Unsupported class"); + } + } \ No newline at end of file From 3b8b45ecea49bb103ab3338caf56d1921c6d3f16 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 9 Aug 2022 14:23:51 -0500 Subject: [PATCH 17/18] Add field sections, record labels, display values being populated --- .../core/actions/tables/QueryAction.java | 5 + .../core/actions/values/QValueFormatter.java | 105 +++++++- .../core/instances/QInstanceEnricher.java | 91 ++++++- .../core/instances/QInstanceValidator.java | 54 ++++ .../qqq/backend/core/model/data/QRecord.java | 59 +++-- .../model/metadata/fields/QFieldType.java | 4 - .../frontend/QFrontendProcessMetaData.java | 14 +- .../frontend/QFrontendTableMetaData.java | 30 ++- .../metadata/processes/QProcessMetaData.java | 37 ++- .../model/metadata/tables/QFieldSection.java | 234 ++++++++++++++++++ .../model/metadata/tables/QTableMetaData.java | 135 ++++++++++ .../core/model/metadata/tables/Tier.java | 33 +++ .../implementations/mock/MockQueryAction.java | 1 - .../actions/values/QValueFormatterTest.java | 160 ++++++++++++ .../instances/QInstanceValidatorTest.java | 125 ++++++++++ .../qqq/backend/module/rdbms/TestUtils.java | 16 ++ .../rdbms/actions/RDBMSQueryActionTest.java | 26 ++ .../sampleapp/SampleMetaDataProvider.java | 52 +++- .../test/resources/prime-test-database.sql | 15 +- 19 files changed, 1140 insertions(+), 56 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QFieldSection.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Tier.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java index c929bc24..c09ce3a9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.tables; import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; @@ -48,6 +49,10 @@ public class QueryAction // todo pre-customization - just get to modify the request? QueryOutput queryOutput = qModule.getQueryInterface().execute(queryInput); // todo post-customization - can do whatever w/ the result if you want + + QValueFormatter.setDisplayValuesInRecords(queryInput.getTable(), queryOutput.getRecords()); + return queryOutput; } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index 107b66bd..4c20ae7f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -23,7 +23,10 @@ package com.kingsrook.qqq.backend.core.actions.values; import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.logging.log4j.LogManager; @@ -63,7 +66,25 @@ public class QValueFormatter } catch(Exception e) { - LOG.warn("Error formatting value [" + value + "] for field [" + field.getName() + "] with format [" + field.getDisplayFormat() + "]: " + e.getMessage()); + try + { + if(e.getMessage().equals("f != java.lang.Integer")) + { + return formatValue(field, ValueUtils.getValueAsBigDecimal(value)); + } + else if(e.getMessage().equals("d != java.math.BigDecimal")) + { + return formatValue(field, ValueUtils.getValueAsInteger(value)); + } + else + { + LOG.warn("Error formatting value [" + value + "] for field [" + field.getName() + "] with format [" + field.getDisplayFormat() + "]: " + e.getMessage()); + } + } + catch(Exception e2) + { + LOG.warn("Caught secondary exception trying to convert type on field [" + field.getName() + "] for formatting", e); + } } } @@ -72,4 +93,86 @@ public class QValueFormatter //////////////////////////////////////// return (ValueUtils.getValueAsString(value)); } + + + + /******************************************************************************* + ** Make a string from a table's recordLabelFormat and fields, for a given record. + *******************************************************************************/ + public static String formatRecordLabel(QTableMetaData table, QRecord record) + { + if(!StringUtils.hasContent(table.getRecordLabelFormat())) + { + return (formatRecordLabelExceptionalCases(table, record)); + } + + /////////////////////////////////////////////////////////////////////// + // get list of values, then pass them to the string formatter method // + /////////////////////////////////////////////////////////////////////// + try + { + List values = table.getRecordLabelFields().stream() + .map(record::getValue) + .map(v -> v == null ? "" : v) + .toList(); + return (table.getRecordLabelFormat().formatted(values.toArray())); + } + catch(Exception e) + { + return (formatRecordLabelExceptionalCases(table, record)); + } + } + + + + /******************************************************************************* + ** Deal with non-happy-path cases for making a record label. + *******************************************************************************/ + private static String formatRecordLabelExceptionalCases(QTableMetaData table, QRecord record) + { + /////////////////////////////////////////////////////////////////////////////////////// + // if there's no record label format, then just return the primary key display value // + /////////////////////////////////////////////////////////////////////////////////////// + String pkeyDisplayValue = record.getDisplayValue(table.getPrimaryKeyField()); + if(StringUtils.hasContent(pkeyDisplayValue)) + { + return (pkeyDisplayValue); + } + + String pkeyRawValue = ValueUtils.getValueAsString(record.getValue(table.getPrimaryKeyField())); + if(StringUtils.hasContent(pkeyRawValue)) + { + return (pkeyRawValue); + } + + /////////////////////////////////////////////////////////////////////////////// + // worst case scenario, return empty string, but never null from this method // + /////////////////////////////////////////////////////////////////////////////// + return (""); + } + + + + /******************************************************************************* + ** For a list of records, set their recordLabels and display values + *******************************************************************************/ + public static void setDisplayValuesInRecords(QTableMetaData table, List records) + { + if(records == null) + { + return; + } + + for(QRecord record : records) + { + for(QFieldMetaData field : table.getFields().values()) + { + String formattedValue = QValueFormatter.formatValue(field, record.getValue(field.getName())); + record.setDisplayValue(field.getName(), formattedValue); + } + + record.setRecordLabel(QValueFormatter.formatRecordLabel(table, record)); + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index de16582d..a7ca02a9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -23,8 +23,10 @@ package com.kingsrook.qqq.backend.core.instances; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -32,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; @@ -41,13 +44,16 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMe import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteStoreStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditReceiveValuesStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditStoreRecordsStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertStoreRecordsStep; import com.kingsrook.qqq.backend.core.processes.implementations.general.LoadInitialRecordsStep; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -111,6 +117,11 @@ public class QInstanceEnricher { table.getFields().values().forEach(this::enrich); } + + if(CollectionUtils.nullSafeIsEmpty(table.getSections())) + { + generateTableFieldSections(table); + } } @@ -196,12 +207,19 @@ public class QInstanceEnricher *******************************************************************************/ private String nameToLabel(String name) { - if(name == null) + if(!StringUtils.hasContent(name)) { - return (null); + return (name); } - return (name.substring(0, 1).toUpperCase(Locale.ROOT) + name.substring(1).replaceAll("([A-Z])", " $1")); + if(name.length() == 1) + { + return (name.substring(0, 1).toUpperCase(Locale.ROOT)); + } + else + { + return (name.substring(0, 1).toUpperCase(Locale.ROOT) + name.substring(1).replaceAll("([A-Z])", " $1")); + } } @@ -422,4 +440,71 @@ public class QInstanceEnricher ))); } + + + /******************************************************************************* + ** If a table didn't have any sections, generate "sensible defaults" + *******************************************************************************/ + private void generateTableFieldSections(QTableMetaData table) + { + if(CollectionUtils.nullSafeIsEmpty(table.getFields())) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // assume this table is invalid if it has no fields, but surely it doesn't need any sections then. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + return; + } + + ////////////////////////////////////////////////////////////////////////////// + // create an identity section for the id and any fields in the record label // + ////////////////////////////////////////////////////////////////////////////// + QFieldSection identitySection = new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, new ArrayList<>()); + + Set usedFieldNames = new HashSet<>(); + + if(StringUtils.hasContent(table.getPrimaryKeyField())) + { + identitySection.getFieldNames().add(table.getPrimaryKeyField()); + usedFieldNames.add(table.getPrimaryKeyField()); + } + + if(CollectionUtils.nullSafeHasContents(table.getRecordLabelFields())) + { + for(String fieldName : table.getRecordLabelFields()) + { + if(!usedFieldNames.contains(fieldName)) + { + identitySection.getFieldNames().add(fieldName); + usedFieldNames.add(fieldName); + } + } + } + + if(!identitySection.getFieldNames().isEmpty()) + { + table.addSection(identitySection); + } + + /////////////////////////////////////////////////////////////////////////////// + // if there are more fields, then add them in a default/Other Fields section // + /////////////////////////////////////////////////////////////////////////////// + QFieldSection otherSection = new QFieldSection("otherFields", "Other Fields", new QIcon("dataset"), Tier.T2, new ArrayList<>()); + if(CollectionUtils.nullSafeHasContents(table.getFields())) + { + for(String fieldName : table.getFields().keySet()) + { + if(!usedFieldNames.contains(fieldName)) + { + otherSection.getFieldNames().add(fieldName); + usedFieldNames.add(fieldName); + } + } + } + + if(!otherSection.getFieldNames().isEmpty()) + { + table.addSection(otherSection); + } + + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index c7665022..9ae87598 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -32,6 +32,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -160,12 +163,63 @@ public class QInstanceValidator } }); } + + ////////////////////////////////////////// + // validate field sections in the table // + ////////////////////////////////////////// + Set fieldNamesInSections = new HashSet<>(); + QFieldSection tier1Section = null; + if(table.getSections() != null) + { + for(QFieldSection section : table.getSections()) + { + validateSection(errors, table, section, fieldNamesInSections); + if(section.getTier().equals(Tier.T1)) + { + assertCondition(errors, tier1Section == null, "Table " + tableName + " has more than 1 section listed as Tier 1"); + tier1Section = section; + } + } + } + + if(CollectionUtils.nullSafeHasContents(table.getFields())) + { + for(String fieldName : table.getFields().keySet()) + { + assertCondition(errors, fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is not listed in any field sections."); + } + } + }); } } + /******************************************************************************* + ** + *******************************************************************************/ + private void validateSection(List errors, QTableMetaData table, QFieldSection section, Set fieldNamesInSections) + { + assertCondition(errors, StringUtils.hasContent(section.getName()), "Missing a name for field section in table " + table.getName() + "."); + assertCondition(errors, StringUtils.hasContent(section.getLabel()), "Missing a label for field section in table " + table.getLabel() + "."); + if(assertCondition(errors, CollectionUtils.nullSafeHasContents(section.getFieldNames()), "Table " + table.getName() + " section " + section.getName() + " does not have any fields.")) + { + if(table.getFields() != null) + { + for(String fieldName : section.getFieldNames()) + { + assertCondition(errors, table.getFields().containsKey(fieldName), "Table " + table.getName() + " section " + section.getName() + " specifies fieldName " + fieldName + ", which is not a field on this table."); + assertCondition(errors, !fieldNamesInSections.contains(fieldName), "Table " + table.getName() + " has field " + fieldName + " listed more than once in its field sections."); + + fieldNamesInSections.add(fieldName); + } + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index 2ef044ff..3c30366f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -55,7 +54,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class QRecord implements Serializable { - private String tableName; + private String tableName; + private String recordLabel; + private Map values = new LinkedHashMap<>(); private Map displayValues = new LinkedHashMap<>(); private Map backendDetails = new LinkedHashMap<>(); @@ -90,6 +91,7 @@ public class QRecord implements Serializable public QRecord(QRecord record) { this.tableName = record.tableName; + this.recordLabel = record.recordLabel; this.values = record.values; this.displayValues = record.displayValues; this.backendDetails = record.backendDetails; @@ -139,15 +141,6 @@ public class QRecord implements Serializable - /******************************************************************************* - ** - *******************************************************************************/ - public void setDisplayValue(QFieldMetaData field, Serializable rawValue) - { - displayValues.put(field.getName(), QValueFormatter.formatValue(field, rawValue)); - } - - /******************************************************************************* ** @@ -160,17 +153,6 @@ public class QRecord implements Serializable - /******************************************************************************* - ** - *******************************************************************************/ - public QRecord withDisplayValue(QFieldMetaData field, Serializable rawValue) - { - setDisplayValue(field, rawValue); - return (this); - } - - - /******************************************************************************* ** Getter for tableName ** @@ -205,6 +187,39 @@ public class QRecord implements Serializable + /******************************************************************************* + ** Getter for recordLabel + ** + *******************************************************************************/ + public String getRecordLabel() + { + return recordLabel; + } + + + + /******************************************************************************* + ** Setter for recordLabel + ** + *******************************************************************************/ + public void setRecordLabel(String recordLabel) + { + this.recordLabel = recordLabel; + } + + + /******************************************************************************* + ** Fluent setter for recordLabel + ** + *******************************************************************************/ + public QRecord withRecordLabel(String recordLabel) + { + this.recordLabel = recordLabel; + return (this); + } + + + /******************************************************************************* ** Getter for values ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java index 79bc824a..324e8991 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java @@ -61,10 +61,6 @@ public enum QFieldType { return (INTEGER); } - if(c.equals(Boolean.class)) - { - return (BOOLEAN); - } if(c.equals(BigDecimal.class)) { return (DECIMAL); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java index c6f86289..f6cb8e0a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java @@ -30,6 +30,7 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -45,6 +46,8 @@ public class QFrontendProcessMetaData private String tableName; private boolean isHidden; + private String iconName; + private List frontendSteps; ////////////////////////////////////////////////////////////////////////////////// @@ -77,6 +80,11 @@ public class QFrontendProcessMetaData frontendSteps = new ArrayList<>(); } } + + if(processMetaData.getIcon() != null && StringUtils.hasContent(processMetaData.getIcon().getName())) + { + this.iconName = processMetaData.getIcon().getName(); + } } @@ -148,12 +156,12 @@ public class QFrontendProcessMetaData /******************************************************************************* - ** Setter for isHidden + ** Getter for iconName ** *******************************************************************************/ - public void setIsHidden(boolean isHidden) + public String getIconName() { - this.isHidden = isHidden; + return iconName; } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java index 0253448f..283402af 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java @@ -23,11 +23,14 @@ package com.kingsrook.qqq.backend.core.model.metadata.frontend; import java.util.HashMap; +import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -43,7 +46,10 @@ public class QFrontendTableMetaData private boolean isHidden; private String primaryKeyField; + private String iconName; + private Map fields; + private List sections; ////////////////////////////////////////////////////////////////////////////////// // do not add setters. take values from the source-object in the constructor!! // @@ -68,6 +74,13 @@ public class QFrontendTableMetaData { this.fields.put(entry.getKey(), new QFrontendFieldMetaData(entry.getValue())); } + + this.sections = tableMetaData.getSections(); + } + + if(tableMetaData.getIcon() != null && StringUtils.hasContent(tableMetaData.getIcon().getName())) + { + this.iconName = tableMetaData.getIcon().getName(); } } @@ -117,6 +130,17 @@ public class QFrontendTableMetaData + /******************************************************************************* + ** Getter for sections + ** + *******************************************************************************/ + public List getSections() + { + return sections; + } + + + /******************************************************************************* ** Getter for isHidden ** @@ -129,11 +153,11 @@ public class QFrontendTableMetaData /******************************************************************************* - ** Setter for isHidden + ** Getter for iconName ** *******************************************************************************/ - public void setIsHidden(boolean isHidden) + public String getIconName() { - this.isHidden = isHidden; + return iconName; } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index a63f66ba..07105a34 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -27,7 +27,7 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; /******************************************************************************* @@ -44,6 +44,7 @@ public class QProcessMetaData implements QAppChildMetaData private List stepList; private String parentAppName; + private QIcon icon; @@ -321,4 +322,38 @@ public class QProcessMetaData implements QAppChildMetaData this.parentAppName = parentAppName; } + + + /******************************************************************************* + ** Getter for icon + ** + *******************************************************************************/ + public QIcon getIcon() + { + return icon; + } + + + + /******************************************************************************* + ** Setter for icon + ** + *******************************************************************************/ + public void setIcon(QIcon icon) + { + this.icon = icon; + } + + + + /******************************************************************************* + ** Fluent setter for icon + ** + *******************************************************************************/ + public QProcessMetaData withIcon(QIcon icon) + { + this.icon = icon; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QFieldSection.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QFieldSection.java new file mode 100644 index 00000000..3324e581 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QFieldSection.java @@ -0,0 +1,234 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; + + +/******************************************************************************* + ** A section of fields - a logical grouping. + *******************************************************************************/ +public class QFieldSection +{ + private String name; + private String label; + private Tier tier; + + private List fieldNames; + private QIcon icon; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QFieldSection() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QFieldSection(String name, String label, QIcon icon, Tier tier, List fieldNames) + { + this.name = name; + this.label = label; + this.icon = icon; + this.tier = tier; + this.fieldNames = fieldNames; + } + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Setter for name + ** + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public QFieldSection withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** Setter for label + ** + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + ** + *******************************************************************************/ + public QFieldSection withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for tier + ** + *******************************************************************************/ + public Tier getTier() + { + return tier; + } + + + + /******************************************************************************* + ** Setter for tier + ** + *******************************************************************************/ + public void setTier(Tier tier) + { + this.tier = tier; + } + + + + /******************************************************************************* + ** Fluent setter for tier + ** + *******************************************************************************/ + public QFieldSection withTier(Tier tier) + { + this.tier = tier; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNames + ** + *******************************************************************************/ + public List getFieldNames() + { + return fieldNames; + } + + + + /******************************************************************************* + ** Setter for fieldNames + ** + *******************************************************************************/ + public void setFieldNames(List fieldNames) + { + this.fieldNames = fieldNames; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNames + ** + *******************************************************************************/ + public QFieldSection withFieldNames(List fieldNames) + { + this.fieldNames = fieldNames; + return (this); + } + + + + /******************************************************************************* + ** Getter for icon + ** + *******************************************************************************/ + public QIcon getIcon() + { + return icon; + } + + + + /******************************************************************************* + ** Setter for icon + ** + *******************************************************************************/ + public void setIcon(QIcon icon) + { + this.icon = icon; + } + + + + /******************************************************************************* + ** Fluent setter for icon + ** + *******************************************************************************/ + public QFieldSection withIcon(QIcon icon) + { + this.icon = icon; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index 506ca3d4..6bb2fdae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables; import java.io.Serializable; +import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -64,6 +65,12 @@ public class QTableMetaData implements QAppChildMetaData, Serializable private String parentAppName; private QIcon icon; + private String recordLabelFormat; + private List recordLabelFields; + + private List sections; + + /******************************************************************************* ** Default constructor. @@ -496,6 +503,7 @@ public class QTableMetaData implements QAppChildMetaData, Serializable } + /******************************************************************************* ** Fluent setter for icon ** @@ -506,4 +514,131 @@ public class QTableMetaData implements QAppChildMetaData, Serializable return (this); } + + + /******************************************************************************* + ** Getter for recordLabelFormat + ** + *******************************************************************************/ + public String getRecordLabelFormat() + { + return recordLabelFormat; + } + + + + /******************************************************************************* + ** Setter for recordLabelFormat + ** + *******************************************************************************/ + public void setRecordLabelFormat(String recordLabelFormat) + { + this.recordLabelFormat = recordLabelFormat; + } + + + + /******************************************************************************* + ** Fluent setter for recordLabelFormat + ** + *******************************************************************************/ + public QTableMetaData withRecordLabelFormat(String recordLabelFormat) + { + this.recordLabelFormat = recordLabelFormat; + return (this); + } + + + + /******************************************************************************* + ** Getter for recordLabelFields + ** + *******************************************************************************/ + public List getRecordLabelFields() + { + return recordLabelFields; + } + + + + /******************************************************************************* + ** Setter for recordLabelFields + ** + *******************************************************************************/ + public void setRecordLabelFields(List recordLabelFields) + { + this.recordLabelFields = recordLabelFields; + } + + + + /******************************************************************************* + ** Fluent setter for recordLabelFields + ** + *******************************************************************************/ + public QTableMetaData withRecordLabelFields(List recordLabelFields) + { + this.recordLabelFields = recordLabelFields; + return (this); + } + + + + /******************************************************************************* + ** Getter for sections + ** + *******************************************************************************/ + public List getSections() + { + return sections; + } + + + + /******************************************************************************* + ** Setter for sections + ** + *******************************************************************************/ + public void setSections(List sections) + { + this.sections = sections; + } + + + + /******************************************************************************* + ** Fluent setter for sections + ** + *******************************************************************************/ + public QTableMetaData withSections(List fieldSections) + { + this.sections = fieldSections; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addSection(QFieldSection fieldSection) + { + if(this.sections == null) + { + this.sections = new ArrayList<>(); + } + this.sections.add(fieldSection); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData withSection(QFieldSection fieldSection) + { + addSection(fieldSection); + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Tier.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Tier.java new file mode 100644 index 00000000..f83a7f7f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/Tier.java @@ -0,0 +1,33 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum Tier +{ + T1, + T2, + T3 +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java index ec5c3365..ff3761f7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java @@ -65,7 +65,6 @@ public class MockQueryAction implements QueryInterface { Serializable value = field.equals("id") ? (i + 1) : getValue(table, field); record.setValue(field, value); - record.setDisplayValue(table.getField(field), value); } queryOutput.addRecord(record); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java new file mode 100644 index 00000000..4be9cf52 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java @@ -0,0 +1,160 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.values; + + +import java.math.BigDecimal; +import java.util.Collections; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for QValueFormatter + *******************************************************************************/ +class QValueFormatterTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFormatValue() + { + assertNull(QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), null)); + + assertEquals("1", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), 1)); + assertEquals("1,000", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), 1000)); + assertEquals("1000", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(null), 1000)); + assertEquals("$1,000.00", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.CURRENCY), 1000)); + assertEquals("1,000.00", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.DECIMAL2_COMMAS), 1000)); + assertEquals("1000.00", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.DECIMAL2), 1000)); + + assertEquals("1", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), new BigDecimal("1"))); + assertEquals("1,000", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), new BigDecimal("1000"))); + assertEquals("1000", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.STRING), new BigDecimal("1000"))); + assertEquals("1000", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.STRING), 1000)); + + ////////////////////////////////////////////////// + // this one flows through the exceptional cases // + ////////////////////////////////////////////////// + assertEquals("1000.01", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), new BigDecimal("1000.01"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFormatRecordLabel() + { + QTableMetaData table = new QTableMetaData().withRecordLabelFormat("%s %s").withRecordLabelFields(List.of("firstName", "lastName")); + assertEquals("Darin Kelkhoff", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff"))); + assertEquals("Darin ", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("firstName", "Darin"))); + assertEquals("Darin ", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("firstName", "Darin").withValue("lastName", null))); + + table = new QTableMetaData().withRecordLabelFormat("%s " + DisplayFormat.CURRENCY).withRecordLabelFields(List.of("firstName", "price")); + assertEquals("Darin $10,000.00", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("firstName", "Darin").withValue("price", new BigDecimal(10000)))); + + table = new QTableMetaData().withRecordLabelFormat(DisplayFormat.DEFAULT).withRecordLabelFields(List.of("id")); + assertEquals("123456", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("id", "123456"))); + + /////////////////////////////////////////////////////// + // exceptional flow: no recordLabelFormat specified // + /////////////////////////////////////////////////////// + table = new QTableMetaData().withPrimaryKeyField("id"); + assertEquals("42", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("id", 42))); + + ///////////////////////////////////////////////// + // exceptional flow: no fields for the format // + ///////////////////////////////////////////////// + table = new QTableMetaData().withRecordLabelFormat("%s %s").withPrimaryKeyField("id"); + assertEquals("128", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("id", 128))); + + ///////////////////////////////////////////////////////// + // exceptional flow: not enough fields for the format // + ///////////////////////////////////////////////////////// + table = new QTableMetaData().withRecordLabelFormat("%s %s").withRecordLabelFields(List.of("a")).withPrimaryKeyField("id"); + assertEquals("256", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("a", 47).withValue("id", 256))); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // exceptional flow (kinda): too many fields for the format (just get the ones that are in the format) // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + table = new QTableMetaData().withRecordLabelFormat("%s %s").withRecordLabelFields(List.of("a", "b", "c")).withPrimaryKeyField("id"); + assertEquals("47 48", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("a", 47).withValue("b", 48).withValue("c", 49).withValue("id", 256))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSetDisplayValuesInRecords() + { + QTableMetaData table = new QTableMetaData() + .withRecordLabelFormat("%s %s") + .withRecordLabelFields(List.of("firstName", "lastName")) + .withField(new QFieldMetaData("firstName", QFieldType.STRING)) + .withField(new QFieldMetaData("lastName", QFieldType.STRING)) + .withField(new QFieldMetaData("price", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY)) + .withField(new QFieldMetaData("quantity", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS)); + + ///////////////////////////////////////////////////////////////// + // first, make sure it doesn't crash with null or empty inputs // + ///////////////////////////////////////////////////////////////// + QValueFormatter.setDisplayValuesInRecords(table, null); + QValueFormatter.setDisplayValuesInRecords(table, Collections.emptyList()); + + List records = List.of( + new QRecord() + .withValue("firstName", "Tim") + .withValue("lastName", "Chamberlain") + .withValue("price", new BigDecimal("3.50")) + .withValue("quantity", 1701), + new QRecord() + .withValue("firstName", "Tyler") + .withValue("lastName", "Samples") + .withValue("price", new BigDecimal("174999.99")) + .withValue("quantity", 47) + ); + + QValueFormatter.setDisplayValuesInRecords(table, records); + + assertEquals("Tim Chamberlain", records.get(0).getRecordLabel()); + assertEquals("$3.50", records.get(0).getDisplayValue("price")); + assertEquals("1,701", records.get(0).getDisplayValue("quantity")); + + assertEquals("Tyler Samples", records.get(1).getRecordLabel()); + assertEquals("$174,999.99", records.get(1).getDisplayValue("price")); + assertEquals("47", records.get(1).getDisplayValue("quantity")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 743f43ef..a5cefc2c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -24,10 +24,17 @@ package com.kingsrook.qqq.backend.core.instances; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.function.Consumer; import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -283,6 +290,124 @@ class QInstanceValidatorTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldSectionsMissingName() + { + QTableMetaData table = new QTableMetaData().withName("test") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withSection(new QFieldSection(null, "Section 1", new QIcon("person"), Tier.T1, List.of("id"))) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "Missing a name"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldSectionsMissingLabel() + { + QTableMetaData table = new QTableMetaData().withName("test") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withSection(new QFieldSection("section1", null, new QIcon("person"), Tier.T1, List.of("id"))) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "Missing a label"); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldSectionsNoFields() + { + QTableMetaData table1 = new QTableMetaData().withName("test") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of())) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + assertValidationFailureReasons((qInstance) -> qInstance.addTable(table1), "section1 does not have any fields", "field id is not listed in any field sections"); + + QTableMetaData table2 = new QTableMetaData().withName("test") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, null)) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + assertValidationFailureReasons((qInstance) -> qInstance.addTable(table2), "section1 does not have any fields", "field id is not listed in any field sections"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldSectionsUnrecognizedFieldName() + { + QTableMetaData table = new QTableMetaData().withName("test") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id", "od"))) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "not a field on this table"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldSectionsDuplicatedFieldName() + { + QTableMetaData table1 = new QTableMetaData().withName("test") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id", "id"))) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + assertValidationFailureReasons((qInstance) -> qInstance.addTable(table1), "more than once"); + + QTableMetaData table2 = new QTableMetaData().withName("test") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id"))) + .withSection(new QFieldSection("section2", "Section 2", new QIcon("person"), Tier.T2, List.of("id"))) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + assertValidationFailureReasons((qInstance) -> qInstance.addTable(table2), "more than once"); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNotInAnySections() + { + QTableMetaData table = new QTableMetaData().withName("test") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id"))) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("name", QFieldType.STRING)); + assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "not listed in any field sections"); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldSectionsMultipleTier1() + { + QTableMetaData table = new QTableMetaData().withName("test") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withSection(new QFieldSection("section1", "Section 1", new QIcon("person"), Tier.T1, List.of("id"))) + .withSection(new QFieldSection("section2", "Section 2", new QIcon("person"), Tier.T1, List.of("name"))) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("name", QFieldType.STRING)); + assertValidationFailureReasons((qInstance) -> qInstance.addTable(table), "more than 1 section listed as Tier 1"); + } + + + /******************************************************************************* ** Run a little setup code on a qInstance; then validate it, and assert that it ** failed validation with reasons that match the supplied vararg-reasons (but allow diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java index 5354578a..96b75742 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java @@ -22,10 +22,12 @@ package com.kingsrook.qqq.backend.module.rdbms; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; @@ -48,11 +50,25 @@ public class TestUtils QInstance qInstance = new QInstance(); qInstance.addBackend(defineBackend()); qInstance.addTable(defineTablePerson()); + qInstance.setAuthentication(defineAuthentication()); return (qInstance); } + /******************************************************************************* + ** Define the authentication used in standard tests - using 'mock' type. + ** + *******************************************************************************/ + public static QAuthenticationMetaData defineAuthentication() + { + return new QAuthenticationMetaData() + .withName("mock") + .withType(QAuthenticationType.MOCK); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java index 5e53c0f2..3e4c37ba 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java @@ -23,16 +23,20 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; /******************************************************************************* @@ -416,7 +420,29 @@ public class RDBMSQueryActionTest extends RDBMSActionTest QueryInput queryInput = new QueryInput(); queryInput.setInstance(TestUtils.defineInstance()); queryInput.setTableName(TestUtils.defineTablePerson().getName()); + queryInput.setSession(new QSession()); return queryInput; } + + + /******************************************************************************* + ** This doesn't really test any RDBMS code, but is a checkpoint that the core + ** module is populating displayValues when it performs the system-level query action. + *******************************************************************************/ + @Test + public void testThatDisplayValuesGetSetGoingThroughQueryAction() throws QException + { + QueryInput queryInput = initQueryRequest(); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(5, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + + for(QRecord record : queryOutput.getRecords()) + { + assertThat(record.getValues()).isNotEmpty(); + assertThat(record.getDisplayValues()).isNotEmpty(); + assertThat(record.getErrors()).isEmpty(); + } + } + } \ No newline at end of file diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index 87458a05..ed82ff44 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -33,16 +33,20 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.general.LoadInitialRecordsStep; import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; @@ -123,18 +127,25 @@ public class SampleMetaDataProvider { qInstance.addApp(new QAppMetaData() .withName(APP_NAME_GREETINGS) - .withChild(qInstance.getProcess(PROCESS_NAME_GREET)) - .withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE))); + .withIcon(new QIcon().withName("emoji_people")) + .withChild(qInstance.getProcess(PROCESS_NAME_GREET) + .withIcon(new QIcon().withName("emoji_people"))) + .withChild(qInstance.getTable(TABLE_NAME_PERSON) + .withIcon(new QIcon().withName("person"))) + .withChild(qInstance.getTable(TABLE_NAME_CITY) + .withIcon(new QIcon().withName("location_city"))) + .withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE)) + .withIcon(new QIcon().withName("waving_hand"))); qInstance.addApp(new QAppMetaData() .withName(APP_NAME_PEOPLE) - .withChild(qInstance.getTable(TABLE_NAME_PERSON)) - .withChild(qInstance.getTable(TABLE_NAME_CITY)) + .withIcon(new QIcon().withName("person")) .withChild(qInstance.getApp(APP_NAME_GREETINGS))); qInstance.addApp(new QAppMetaData() .withName(APP_NAME_MISCELLANEOUS) - .withChild(qInstance.getTable(TABLE_NAME_CARRIER)) + .withIcon(new QIcon().withName("stars")) + .withChild(qInstance.getTable(TABLE_NAME_CARRIER).withIcon(new QIcon("local_shipping"))) .withChild(qInstance.getProcess(PROCESS_NAME_SIMPLE_SLEEP)) .withChild(qInstance.getProcess(PROCESS_NAME_SLEEP_INTERACTIVE)) .withChild(qInstance.getProcess(PROCESS_NAME_SIMPLE_THROW))); @@ -205,19 +216,25 @@ public class SampleMetaDataProvider table.setName(TABLE_NAME_CARRIER); table.setBackendName(RDBMS_BACKEND_NAME); table.setPrimaryKeyField("id"); + table.setRecordLabelFormat("%s"); + table.setRecordLabelFields(List.of("name")); table.addField(new QFieldMetaData("id", QFieldType.INTEGER)); table.addField(new QFieldMetaData("name", QFieldType.STRING) .withIsRequired(true)); - table.addField(new QFieldMetaData("company_code", QFieldType.STRING) // todo enum + table.addField(new QFieldMetaData("company_code", QFieldType.STRING) // todo PVS .withLabel("Company") .withIsRequired(true) .withBackendName("comp_code")); - table.addField(new QFieldMetaData("service_level", QFieldType.STRING) - .withIsRequired(true)); // todo enum + table.addField(new QFieldMetaData("service_level", QFieldType.STRING) // todo PVS + .withLabel("Service Level") + .withIsRequired(true)); + + table.addSection(new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, List.of("id", "name"))); + table.addSection(new QFieldSection("basicInfo", "Basic Info", new QIcon("dataset"), Tier.T2, List.of("company_code", "service_level"))); return (table); } @@ -234,13 +251,24 @@ public class SampleMetaDataProvider .withLabel("Person") .withBackendName(RDBMS_BACKEND_NAME) .withPrimaryKeyField("id") - .withField(new QFieldMetaData("id", QFieldType.INTEGER)) - .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) - .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) + .withRecordLabelFormat("%s %s") + .withRecordLabelFields(List.of("firstName", "lastName")) + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date").withIsEditable(false)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date").withIsEditable(false)) .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name").withIsRequired(true)) .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name").withIsRequired(true)) .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) - .withField(new QFieldMetaData("email", QFieldType.STRING)); + .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withField(new QFieldMetaData("annualSalary", QFieldType.DECIMAL).withBackendName("annual_salary").withDisplayFormat(DisplayFormat.CURRENCY)) + .withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER).withBackendName("days_worked").withDisplayFormat(DisplayFormat.COMMAS)) + + .withSection(new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, List.of("id", "firstName", "lastName"))) + .withSection(new QFieldSection("basicInfo", "Basic Info", new QIcon("dataset"), Tier.T2, List.of("email", "birthDate"))) + .withSection(new QFieldSection("employmentInfo", "Employment Info", new QIcon("work"), Tier.T2, List.of("annualSalary", "daysWorked"))) + .withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))) + + .withInferredFieldBackendNames(); } diff --git a/qqq-sample-project/src/test/resources/prime-test-database.sql b/qqq-sample-project/src/test/resources/prime-test-database.sql index 07ab6ac6..ef295c31 100644 --- a/qqq-sample-project/src/test/resources/prime-test-database.sql +++ b/qqq-sample-project/src/test/resources/prime-test-database.sql @@ -29,14 +29,17 @@ CREATE TABLE person first_name VARCHAR(80) NOT NULL, last_name VARCHAR(80) NOT NULL, birth_date DATE, - email VARCHAR(250) NOT NULL + email VARCHAR(250) NOT NULL, + + annual_salary DECIMAL(12, 2), + days_worked INTEGER ); -INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com'); -INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com'); -INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com'); -INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com'); -INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com'); +INSERT INTO person (id, first_name, last_name, birth_date, email, annual_salary, days_worked) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com', 75003.50, 1001); +INSERT INTO person (id, first_name, last_name, birth_date, email, annual_salary, days_worked) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com', 150000, 10100); +INSERT INTO person (id, first_name, last_name, birth_date, email, annual_salary, days_worked) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com', 300000, 100100); +INSERT INTO person (id, first_name, last_name, birth_date, email, annual_salary, days_worked) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com', 950000, 75); +INSERT INTO person (id, first_name, last_name, birth_date, email, annual_salary, days_worked) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com', 1500000, 1); DROP TABLE IF EXISTS carrier; CREATE TABLE carrier From 3b1f0b47c13325850326145405b7434a1c7020f7 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 11 Aug 2022 09:42:30 -0500 Subject: [PATCH 18/18] Feedback from code reviews --- .../backend/core/actions/ActionHelper.java | 5 +- .../core/actions/QBackendTransaction.java | 21 +-- .../core/actions/metadata/MetaDataAction.java | 17 +- .../core/actions/tables/QueryAction.java | 5 +- .../core/instances/QInstanceEnricher.java | 12 +- .../actions/metadata/MetaDataOutput.java | 7 +- .../qqq/backend/core/model/data/QField.java | 18 ++- .../model/metadata/fields/QFieldMetaData.java | 11 +- .../model/metadata/frontend/AppTreeNode.java | 152 ++++++++++++++++++ .../metadata/frontend/AppTreeNodeType.java | 2 +- .../frontend/QFrontendAppMetaData.java | 44 +---- .../frontend/QFrontendProcessMetaData.java | 3 +- .../frontend/QFrontendTableMetaData.java | 3 +- .../metadata/layout/QAppChildMetaData.java | 1 + .../model/metadata/layout/QAppMetaData.java | 3 +- .../model/metadata/tables/QTableMetaData.java | 12 -- .../etl/basic/BasicETLLoadFunction.java | 2 +- .../etl/streamed/StreamedETLBackendStep.java | 13 +- .../qqq/backend/core/utils/ValueUtils.java | 7 +- .../actions/metadata/MetaDataActionTest.java | 11 +- .../core/model/data/QRecordEntityTest.java | 3 + .../core/model/data/testentities/Item.java | 5 +- .../Auth0AuthenticationModuleTest.java | 25 +-- .../etl/streamed/StreamedETLProcessTest.java | 11 +- .../backend/core/utils/ValueUtilsTest.java | 33 ++-- .../rdbms/actions/RDBMSInsertAction.java | 2 - .../rdbms/actions/RDBMSTransaction.java | 19 +-- .../module/rdbms/jdbc/QueryManager.java | 36 ++++- .../qqq/backend/module/rdbms/TestUtils.java | 31 ++++ .../module/rdbms/actions/RDBMSActionTest.java | 28 +--- .../rdbms/actions/RDBMSDeleteActionTest.java | 2 +- .../rdbms/actions/RDBMSTransactionTest.java | 96 +++++++++++ .../module/rdbms/jdbc/QueryManagerTest.java | 31 ++++ .../sampleapp/SampleMetaDataProvider.java | 11 +- 34 files changed, 486 insertions(+), 196 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNode.java create mode 100644 qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransactionTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java index 11ef0eeb..b1662c6a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java @@ -30,13 +30,10 @@ import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModu /******************************************************************************* - ** + ** Utility methods to be shared by all of the various Actions (e.g., InsertAction) *******************************************************************************/ public class ActionHelper { - private int f; - - /******************************************************************************* ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java index 4220ec90..64a0c57e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java @@ -22,7 +22,6 @@ package com.kingsrook.qqq.backend.core.actions; -import java.io.IOException; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -31,12 +30,14 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; ** part of a transaction. ** ** Most obvious use-case would be a JDBC Connection. See subclass in rdbms module. + ** + ** Note: One would imagine that this class shouldn't ever implement Serializable... *******************************************************************************/ public class QBackendTransaction { /******************************************************************************* - ** + ** Commit the transaction. *******************************************************************************/ public void commit() throws QException { @@ -48,7 +49,7 @@ public class QBackendTransaction /******************************************************************************* - ** + ** Rollback the transaction. *******************************************************************************/ public void rollback() throws QException { @@ -60,18 +61,8 @@ public class QBackendTransaction /******************************************************************************* - * Closes this stream and releases any system resources associated - * with it. If the stream is already closed then invoking this - * method has no effect. - * - *

As noted in {@link AutoCloseable#close()}, cases where the - * close may fail require careful attention. It is strongly advised - * to relinquish the underlying resources and to internally - * mark the {@code Closeable} as closed, prior to throwing - * the {@code IOException}. - * - * @throws IOException - * if an I/O error occurs + ** Close any resources associated with the transaction. In theory, should only + ** be called after a commit or rollback was done. *******************************************************************************/ public void close() { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java index f15e319c..ce21b93f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java @@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData; @@ -55,7 +56,7 @@ public class MetaDataAction // todo pre-customization - just get to modify the request? MetaDataOutput metaDataOutput = new MetaDataOutput(); - Map treeNodes = new LinkedHashMap<>(); + Map treeNodes = new LinkedHashMap<>(); ///////////////////////////////////// // map tables to frontend metadata // @@ -64,7 +65,7 @@ public class MetaDataAction for(Map.Entry entry : metaDataInput.getInstance().getTables().entrySet()) { tables.put(entry.getKey(), new QFrontendTableMetaData(entry.getValue(), false)); - treeNodes.put(entry.getKey(), new QFrontendAppMetaData(entry.getValue())); + treeNodes.put(entry.getKey(), new AppTreeNode(entry.getValue())); } metaDataOutput.setTables(tables); @@ -75,7 +76,7 @@ public class MetaDataAction for(Map.Entry entry : metaDataInput.getInstance().getProcesses().entrySet()) { processes.put(entry.getKey(), new QFrontendProcessMetaData(entry.getValue(), false)); - treeNodes.put(entry.getKey(), new QFrontendAppMetaData(entry.getValue())); + treeNodes.put(entry.getKey(), new AppTreeNode(entry.getValue())); } metaDataOutput.setProcesses(processes); @@ -86,11 +87,11 @@ public class MetaDataAction for(Map.Entry entry : metaDataInput.getInstance().getApps().entrySet()) { apps.put(entry.getKey(), new QFrontendAppMetaData(entry.getValue())); - treeNodes.put(entry.getKey(), new QFrontendAppMetaData(entry.getValue())); + treeNodes.put(entry.getKey(), new AppTreeNode(entry.getValue())); for(QAppChildMetaData child : entry.getValue().getChildren()) { - apps.get(entry.getKey()).addChild(new QFrontendAppMetaData(child)); + apps.get(entry.getKey()).addChild(new AppTreeNode(child)); } } metaDataOutput.setApps(apps); @@ -98,7 +99,7 @@ public class MetaDataAction //////////////////////////////////////////////// // organize app tree nodes by their hierarchy // //////////////////////////////////////////////// - List appTree = new ArrayList<>(); + List appTree = new ArrayList<>(); for(QAppMetaData appMetaData : metaDataInput.getInstance().getApps().values()) { if(appMetaData.getParentAppName() == null) @@ -118,9 +119,9 @@ public class MetaDataAction /******************************************************************************* ** *******************************************************************************/ - private void buildAppTree(Map treeNodes, List nodeList, QAppChildMetaData childMetaData) + private void buildAppTree(Map treeNodes, List nodeList, QAppChildMetaData childMetaData) { - QFrontendAppMetaData treeNode = treeNodes.get(childMetaData.getName()); + AppTreeNode treeNode = treeNodes.get(childMetaData.getName()); if(treeNode == null) { return; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java index c09ce3a9..eec134f0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java @@ -50,7 +50,10 @@ public class QueryAction QueryOutput queryOutput = qModule.getQueryInterface().execute(queryInput); // todo post-customization - can do whatever w/ the result if you want - QValueFormatter.setDisplayValuesInRecords(queryInput.getTable(), queryOutput.getRecords()); + if (queryInput.getRecordPipe() == null) + { + QValueFormatter.setDisplayValuesInRecords(queryInput.getTable(), queryOutput.getRecords()); + } return queryOutput; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 278358fa..6104f9db 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -447,7 +447,6 @@ public class QInstanceEnricher /******************************************************************************* -<<<<<<< HEAD ** for all fields in a table, set their backendName, using the default "inference" logic ** see {@link #inferBackendName(String)} *******************************************************************************/ @@ -482,6 +481,17 @@ public class QInstanceEnricher /******************************************************************************* ** Do a default mapping from a camelCase field name to an underscore_style ** name for a backend. + ** + ** Examples: + **

    + **
  • wordAnotherWordMoreWords -> word_another_word_more_words
    • + **
  • lUlUlUl -> l_ul_ul_ul
    • + **
  • StartsUpper -> starts_upper
    • + **
  • TLAFirst -> tla_first
    • + **
  • wordThenTLAInMiddle -> word_then_tla_in_middle
    • + **
  • endWithTLA -> end_with_tla
    • + **
  • TLAAndAnotherTLA -> tla_and_another_tla
    • + **
*******************************************************************************/ static String inferBackendName(String fieldName) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java index f47cc424..759ba708 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.metadata; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData; @@ -40,7 +41,7 @@ public class MetaDataOutput extends AbstractActionOutput private Map processes; private Map apps; - private List appTree; + private List appTree; @@ -93,7 +94,7 @@ public class MetaDataOutput extends AbstractActionOutput ** Getter for appTree ** *******************************************************************************/ - public List getAppTree() + public List getAppTree() { return appTree; } @@ -104,7 +105,7 @@ public class MetaDataOutput extends AbstractActionOutput ** Setter for appTree ** *******************************************************************************/ - public void setAppTree(List appTree) + public void setAppTree(List appTree) { this.appTree = appTree; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java index cbdb9616..6fd3eeb6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java @@ -26,17 +26,22 @@ import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; /******************************************************************************* + ** Annotation to place onto fields in a QRecordEntity, to add additional attributes + ** for propagating down into the corresponding QFieldMetaData ** *******************************************************************************/ @Target(ElementType.FIELD) @Retention(RetentionPolicy.RUNTIME) public @interface QField { + /******************************************************************************* + ** + *******************************************************************************/ + String label() default ""; + /******************************************************************************* ** *******************************************************************************/ @@ -51,4 +56,13 @@ public @interface QField ** *******************************************************************************/ boolean isEditable() default true; + + /******************************************************************************* + ** + *******************************************************************************/ + String displayFormat() default ""; + + ////////////////////////////////////////////////////////////////////////////////////////// + // new attributes here likely need implementation in QFieldMetaData.constructFromGetter // + ////////////////////////////////////////////////////////////////////////////////////////// } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java index 066d2aca..8967c579 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java @@ -28,7 +28,6 @@ import java.util.Optional; import com.github.hervian.reflection.Fun; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QField; -import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -120,10 +119,20 @@ public class QFieldMetaData setIsRequired(fieldAnnotation.isRequired()); setIsEditable(fieldAnnotation.isEditable()); + if(StringUtils.hasContent(fieldAnnotation.label())) + { + setLabel(fieldAnnotation.label()); + } + if(StringUtils.hasContent(fieldAnnotation.backendName())) { setBackendName(fieldAnnotation.backendName()); } + + if(StringUtils.hasContent(fieldAnnotation.displayFormat())) + { + setDisplayFormat(fieldAnnotation.displayFormat()); + } } } catch(QException qe) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNode.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNode.java new file mode 100644 index 00000000..5b912094 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNode.java @@ -0,0 +1,152 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.frontend; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Frontend-version of objects that are parts of the app-hierarchy/tree. + ** e.g., Tables, Processes, and Apps themselves (since they can be nested). + ** + ** These objects are organized into a tree - where each Node can have 0 or more + ** other Nodes as children. + *******************************************************************************/ +public class AppTreeNode +{ + private AppTreeNodeType type; + private String name; + private String label; + private List children; + + private String iconName; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public AppTreeNode(QAppChildMetaData appChildMetaData) + { + this.name = appChildMetaData.getName(); + this.label = appChildMetaData.getLabel(); + + if(appChildMetaData.getClass().equals(QTableMetaData.class)) + { + this.type = AppTreeNodeType.TABLE; + } + else if(appChildMetaData.getClass().equals(QProcessMetaData.class)) + { + this.type = AppTreeNodeType.PROCESS; + } + else if(appChildMetaData.getClass().equals(QAppMetaData.class)) + { + this.type = AppTreeNodeType.APP; + children = new ArrayList<>(); + } + else + { + throw (new IllegalStateException("Unrecognized class for app child meta data: " + appChildMetaData.getClass())); + } + + if(appChildMetaData.getIcon() != null) + { + // todo - propagate icons from parents, if they aren't set here... + this.iconName = appChildMetaData.getIcon().getName(); + } + } + + + + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public AppTreeNodeType getType() + { + return type; + } + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** Getter for children + ** + *******************************************************************************/ + public List getChildren() + { + return children; + } + + + + /******************************************************************************* + ** Getter for iconName + ** + *******************************************************************************/ + public String getIconName() + { + return iconName; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addChild(AppTreeNode childTreeNode) + { + if(children == null) + { + children = new ArrayList<>(); + } + children.add(childTreeNode); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNodeType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNodeType.java index 09c954f4..3b2c7eaf 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNodeType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/AppTreeNodeType.java @@ -23,7 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.frontend; /******************************************************************************* - ** + ** Type for an Node in the an app tree. *******************************************************************************/ public enum AppTreeNodeType { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java index 85d9d885..6f792f97 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java @@ -27,10 +27,6 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -40,12 +36,10 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; @JsonInclude(Include.NON_NULL) public class QFrontendAppMetaData { - private AppTreeNodeType type; - private String name; private String label; - private List children = new ArrayList<>(); + private List children = new ArrayList<>(); private String iconName; @@ -59,24 +53,7 @@ public class QFrontendAppMetaData this.name = appChildMetaData.getName(); this.label = appChildMetaData.getLabel(); - if(appChildMetaData.getClass().equals(QTableMetaData.class)) - { - this.type = AppTreeNodeType.TABLE; - } - else if(appChildMetaData.getClass().equals(QProcessMetaData.class)) - { - this.type = AppTreeNodeType.PROCESS; - } - else if(appChildMetaData.getClass().equals(QAppMetaData.class)) - { - this.type = AppTreeNodeType.APP; - } - else - { - throw (new IllegalStateException("Unrecognized class for app child meta data: " + appChildMetaData.getClass())); - } - - if(appChildMetaData.getIcon() != null && StringUtils.hasContent(appChildMetaData.getIcon().getName())) + if(appChildMetaData.getIcon() != null) { this.iconName = appChildMetaData.getIcon().getName(); } @@ -106,22 +83,11 @@ public class QFrontendAppMetaData - /******************************************************************************* - ** Getter for type - ** - *******************************************************************************/ - public AppTreeNodeType getType() - { - return type; - } - - - /******************************************************************************* ** Getter for children ** *******************************************************************************/ - public List getChildren() + public List getChildren() { return children; } @@ -153,12 +119,12 @@ public class QFrontendAppMetaData /******************************************************************************* ** *******************************************************************************/ - public void addChild(QFrontendAppMetaData qFrontendAppMetaData) + public void addChild(AppTreeNode childAppTreeNode) { if(children == null) { children = new ArrayList<>(); } - children.add(qFrontendAppMetaData); + children.add(childAppTreeNode); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java index f6cb8e0a..182d1b9b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java @@ -30,7 +30,6 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -81,7 +80,7 @@ public class QFrontendProcessMetaData } } - if(processMetaData.getIcon() != null && StringUtils.hasContent(processMetaData.getIcon().getName())) + if(processMetaData.getIcon() != null) { this.iconName = processMetaData.getIcon().getName(); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java index 283402af..3ad7774e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java @@ -30,7 +30,6 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -78,7 +77,7 @@ public class QFrontendTableMetaData this.sections = tableMetaData.getSections(); } - if(tableMetaData.getIcon() != null && StringUtils.hasContent(tableMetaData.getIcon().getName())) + if(tableMetaData.getIcon() != null) { this.iconName = tableMetaData.getIcon().getName(); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppChildMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppChildMetaData.java index 85db26ef..088aefc0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppChildMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppChildMetaData.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.layout; /******************************************************************************* ** Interface shared by meta-data objects which can be placed into an App. + ** e.g., Tables, Processes, and Apps themselves (since they can be nested) *******************************************************************************/ public interface QAppChildMetaData { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java index 22681a47..36ac02d6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java @@ -27,7 +27,8 @@ import java.util.List; /******************************************************************************* - ** + ** MetaData definition of an App - an entity that organizes tables & processes + ** and can be arranged hierarchically (e.g, apps can contain other apps). *******************************************************************************/ public class QAppMetaData implements QAppChildMetaData { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index 854e4b7c..0fec3d48 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -30,7 +30,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; import com.kingsrook.qqq.backend.core.model.data.QRecordEntityField; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; @@ -467,17 +466,6 @@ public class QTableMetaData implements QAppChildMetaData, Serializable - /******************************************************************************* - ** - *******************************************************************************/ - public QTableMetaData withInferredFieldBackendNames() - { - QInstanceEnricher.setInferredFieldBackendNames(this); - return (this); - } - - - /******************************************************************************* ** Getter for parentAppName ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java index 5840c24e..f8e782f2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java @@ -46,7 +46,7 @@ public class BasicETLLoadFunction implements BackendStep private static final Logger LOG = LogManager.getLogger(BasicETLLoadFunction.class); private QBackendTransaction transaction; - private boolean returnStoredRecords = false; + private boolean returnStoredRecords = true; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java index bb4337dc..ce2cfd7b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java @@ -78,12 +78,12 @@ public class StreamedETLBackendStep implements BackendStep // run the query action as an async job // ////////////////////////////////////////// AsyncJobManager asyncJobManager = new AsyncJobManager(); - String queryJobUUID = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> + String queryJobUUID = asyncJobManager.startJob("StreamedETL>QueryAction", (status) -> { basicETLExtractFunction.run(runBackendStepInput, runBackendStepOutput); return (runBackendStepOutput); }); - LOG.info("Started query job [" + queryJobUUID + "] for report"); + LOG.info("Started query job [" + queryJobUUID + "] for streamed ETL"); AsyncJobState queryJobState = AsyncJobState.RUNNING; AsyncJobStatus asyncJobStatus = null; @@ -141,6 +141,14 @@ public class StreamedETLBackendStep implements BackendStep LOG.info("Query job [" + queryJobUUID + "] for ETL completed with status: " + asyncJobStatus); + ///////////////////////////////////////// + // propagate errors from the query job // + ///////////////////////////////////////// + if(asyncJobStatus.getState().equals(AsyncJobState.ERROR)) + { + throw (new QException("Query job failed with an error", asyncJobStatus.getCaughtException())); + } + ////////////////////////////////////////////////////// // send the final records to transform & load steps // ////////////////////////////////////////////////////// @@ -207,6 +215,7 @@ public class StreamedETLBackendStep implements BackendStep runBackendStepInput.setRecords(runBackendStepOutput.getRecords()); BasicETLLoadFunction basicETLLoadFunction = new BasicETLLoadFunction(); + basicETLLoadFunction.setReturnStoredRecords(false); basicETLLoadFunction.setTransaction(transaction); basicETLLoadFunction.run(runBackendStepInput, runBackendStepOutput); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index a33abaa1..08ecf054 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -42,8 +42,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QValueException; *******************************************************************************/ public class ValueUtils { - private static final DateTimeFormatter yyyyMMddWithDashesFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - private static final DateTimeFormatter MdyyyyWithSlashesFormatter = DateTimeFormatter.ofPattern("M/d/yyyy"); + private static final DateTimeFormatter dateTimeFormatter_yyyyMMddWithDashes = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + private static final DateTimeFormatter dateTimeFormatter_MdyyyyWithSlashes = DateTimeFormatter.ofPattern("M/d/yyyy"); @@ -262,7 +262,7 @@ public class ValueUtils private static LocalDate tryLocalDateParsers(String s) { DateTimeParseException lastException = null; - for(DateTimeFormatter dateTimeFormatter : List.of(yyyyMMddWithDashesFormatter, MdyyyyWithSlashesFormatter)) + for(DateTimeFormatter dateTimeFormatter : List.of(dateTimeFormatter_yyyyMMddWithDashes, dateTimeFormatter_MdyyyyWithSlashes)) { try { @@ -386,7 +386,6 @@ public class ValueUtils } else if(value instanceof Calendar c) { - TimeZone tz = c.getTimeZone(); return (c.toInstant()); } else if(value instanceof LocalDateTime ldt) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java index c86ef97e..c22da8a5 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataActionTest.java @@ -30,6 +30,7 @@ import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput; import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; @@ -84,7 +85,7 @@ class MetaDataActionTest QFrontendAppMetaData peopleApp = apps.get(TestUtils.APP_NAME_PEOPLE); assertThat(peopleApp.getChildren()).isNotEmpty(); - Optional greetingsAppUnderPeopleFromMapOptional = peopleApp.getChildren().stream() + Optional greetingsAppUnderPeopleFromMapOptional = peopleApp.getChildren().stream() .filter(e -> e.getName().equals(TestUtils.APP_NAME_GREETINGS)).findFirst(); assertThat(greetingsAppUnderPeopleFromMapOptional).isPresent(); @@ -97,18 +98,18 @@ class MetaDataActionTest /////////////////////////////////////////////// // assert against the hierarchical apps tree // /////////////////////////////////////////////// - List appTree = result.getAppTree(); - Set appNamesInTopOfTree = appTree.stream().map(QFrontendAppMetaData::getName).collect(Collectors.toSet()); + List appTree = result.getAppTree(); + Set appNamesInTopOfTree = appTree.stream().map(AppTreeNode::getName).collect(Collectors.toSet()); assertThat(appNamesInTopOfTree).contains(TestUtils.APP_NAME_PEOPLE); assertThat(appNamesInTopOfTree).contains(TestUtils.APP_NAME_MISCELLANEOUS); assertThat(appNamesInTopOfTree).doesNotContain(TestUtils.APP_NAME_GREETINGS); - Optional peopleAppOptional = appTree.stream() + Optional peopleAppOptional = appTree.stream() .filter(e -> e.getName().equals(TestUtils.APP_NAME_PEOPLE)).findFirst(); assertThat(peopleAppOptional).isPresent(); assertThat(peopleAppOptional.get().getChildren()).isNotEmpty(); - Optional greetingsAppUnderPeopleFromTree = peopleAppOptional.get().getChildren().stream() + Optional greetingsAppUnderPeopleFromTree = peopleAppOptional.get().getChildren().stream() .filter(e -> e.getName().equals(TestUtils.APP_NAME_GREETINGS)).findFirst(); assertThat(greetingsAppUnderPeopleFromTree).isPresent(); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java index be3cd897..2904fbd2 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java @@ -26,6 +26,7 @@ import java.math.BigDecimal; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.testentities.Item; import com.kingsrook.qqq.backend.core.model.data.testentities.ItemWithPrimitives; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -177,6 +178,8 @@ class QRecordEntityTest /////////////////////////////////////////////////////////////// // assert about attributes that came from @QField annotation // /////////////////////////////////////////////////////////////// + assertEquals("SKU", qTableMetaData.getField("sku").getLabel()); + assertEquals(DisplayFormat.COMMAS, qTableMetaData.getField("quantity").getDisplayFormat()); assertTrue(qTableMetaData.getField("sku").getIsRequired()); assertFalse(qTableMetaData.getField("quantity").getIsEditable()); assertEquals("is_featured", qTableMetaData.getField("featured").getBackendName()); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java index 86653880..fd5225a0 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.data.testentities; import java.math.BigDecimal; import com.kingsrook.qqq.backend.core.model.data.QField; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; /******************************************************************************* @@ -32,13 +33,13 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; *******************************************************************************/ public class Item extends QRecordEntity { - @QField(isRequired = true) + @QField(isRequired = true, label = "SKU") private String sku; @QField() private String description; - @QField(isEditable = false) + @QField(isEditable = false, displayFormat = DisplayFormat.COMMAS) private Integer quantity; private BigDecimal price; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java index 049d3c57..bc69be9d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/Auth0AuthenticationModuleTest.java @@ -60,21 +60,8 @@ public class Auth0AuthenticationModuleTest /******************************************************************************* - ** Test an expired token where 'now' is set to a time that would not require it to be - ** re-checked, so it'll show as valid - ** - *******************************************************************************/ - @Test - public void testLastTimeCheckedNow() - { - assertTrue(testLastTimeChecked(Instant.now(), UNDECODABLE_TOKEN), "A session just checked 'now' should always be valid"); - } - - - - /******************************************************************************* - ** Test an expired token where 'now' is set to a time that would not require it to be - ** re-checked, so it'll show as valid + ** Test a token where last-checked is set to a time that would not require it to be + ** re-checked, so it'll show as valid no matter what the token is. ** *******************************************************************************/ @Test @@ -87,8 +74,8 @@ public class Auth0AuthenticationModuleTest /******************************************************************************* - ** Test an expired token where 'now' is set to a time that would require it to be - ** re-checked + ** Test a token where last-checked is set to a time that would require it to be + ** re-checked, so it'll show as invalid. ** *******************************************************************************/ @Test @@ -101,8 +88,8 @@ public class Auth0AuthenticationModuleTest /******************************************************************************* - ** Test an expired token where 'now' is set to a time that would require it to be - ** re-checked + ** Test a token where last-checked is past the threshold, so it'll get re-checked, + ** and will fail. ** *******************************************************************************/ @Test diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcessTest.java index 97d756c9..ea09adba 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcessTest.java @@ -55,7 +55,10 @@ class StreamedETLProcessTest RunProcessOutput result = new RunProcessAction().execute(request); assertNotNull(result); - assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process"); + /////////////////////////////////////////////////////////////////////// + // since this is streamed, assert there are no records in the output // + /////////////////////////////////////////////////////////////////////// + assertTrue(result.getRecords().isEmpty()); assertTrue(result.getException().isEmpty()); } @@ -77,12 +80,14 @@ class StreamedETLProcessTest // define our mapping from destination-table field names to source-table field names // /////////////////////////////////////////////////////////////////////////////////////// QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping().withMapping("name", "firstName"); - // request.addValue(StreamedETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping.getMapping())); request.addValue(StreamedETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping)); RunProcessOutput result = new RunProcessAction().execute(request); assertNotNull(result); - assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process"); + /////////////////////////////////////////////////////////////////////// + // since this is streamed, assert there are no records in the output // + /////////////////////////////////////////////////////////////////////// + assertTrue(result.getRecords().isEmpty()); assertTrue(result.getException().isEmpty()); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java index 0ef501a7..998fae6d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java @@ -53,7 +53,6 @@ class ValueUtilsTest @Test void testGetValueAsString() throws QValueException { - //noinspection ConstantConditions assertNull(ValueUtils.getValueAsString(null)); assertEquals("", ValueUtils.getValueAsString("")); assertEquals(" ", ValueUtils.getValueAsString(" ")); @@ -164,26 +163,28 @@ class ValueUtilsTest @Test void testGetValueAsLocalDate() throws QValueException { + LocalDate expected = LocalDate.of(1980, Month.MAY, 31); + assertNull(ValueUtils.getValueAsLocalDate(null)); assertNull(ValueUtils.getValueAsLocalDate("")); assertNull(ValueUtils.getValueAsLocalDate(" ")); - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(LocalDate.of(1980, 5, 31))); - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new java.sql.Date(80, 4, 31))); + assertEquals(expected, ValueUtils.getValueAsLocalDate(LocalDate.of(1980, 5, 31))); + assertEquals(expected, ValueUtils.getValueAsLocalDate(new java.sql.Date(80, 4, 31))); //noinspection MagicConstant - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new java.util.Date(80, 4, 31))); - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31))); - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31, 12, 0))); - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31, 4, 0))); - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31, 22, 0))); + assertEquals(expected, ValueUtils.getValueAsLocalDate(new java.util.Date(80, 4, 31))); + assertEquals(expected, ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31))); + assertEquals(expected, ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31, 12, 0))); + assertEquals(expected, ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31, 4, 0))); + assertEquals(expected, ValueUtils.getValueAsLocalDate(new java.util.Date(80, Calendar.MAY, 31, 22, 0))); //noinspection MagicConstant - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new GregorianCalendar(1980, 4, 31))); - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(new GregorianCalendar(1980, Calendar.MAY, 31))); - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, 5, 31, 12, 0))); - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, 5, 31, 4, 0))); - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, 5, 31, 22, 0))); - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, Month.MAY, 31, 12, 0))); - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate("1980-05-31")); - assertEquals(LocalDate.of(1980, Month.MAY, 31), ValueUtils.getValueAsLocalDate("05/31/1980")); + assertEquals(expected, ValueUtils.getValueAsLocalDate(new GregorianCalendar(1980, 4, 31))); + assertEquals(expected, ValueUtils.getValueAsLocalDate(new GregorianCalendar(1980, Calendar.MAY, 31))); + assertEquals(expected, ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, 5, 31, 12, 0))); + assertEquals(expected, ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, 5, 31, 4, 0))); + assertEquals(expected, ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, 5, 31, 22, 0))); + assertEquals(expected, ValueUtils.getValueAsLocalDate(LocalDateTime.of(1980, Month.MAY, 31, 12, 0))); + assertEquals(expected, ValueUtils.getValueAsLocalDate("1980-05-31")); + assertEquals(expected, ValueUtils.getValueAsLocalDate("05/31/1980")); assertThrows(QValueException.class, () -> ValueUtils.getValueAsLocalDate("a")); assertThrows(QValueException.class, () -> ValueUtils.getValueAsLocalDate("a,b")); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index b9e76476..7bdd715b 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -172,8 +172,6 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte { LOG.info("Opening transaction"); Connection connection = getConnection(insertInput); - connection.setAutoCommit(false); - System.out.println("Set connection [" + connection + "] to auto-commit false"); return (new RDBMSTransaction(connection)); } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransaction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransaction.java index 529f69ff..0c7e50cf 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransaction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransaction.java @@ -22,8 +22,8 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; -import java.io.IOException; import java.sql.Connection; +import java.sql.SQLException; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.exceptions.QException; import org.apache.logging.log4j.LogManager; @@ -46,8 +46,9 @@ public class RDBMSTransaction extends QBackendTransaction /******************************************************************************* ** *******************************************************************************/ - public RDBMSTransaction(Connection connection) + public RDBMSTransaction(Connection connection) throws SQLException { + connection.setAutoCommit(false); this.connection = connection; } @@ -73,7 +74,6 @@ public class RDBMSTransaction extends QBackendTransaction try { RDBMSTransaction.LOG.info("Committing transaction"); - System.out.println("Calling commit on connection [" + connection + "]"); connection.commit(); RDBMSTransaction.LOG.info("Commit complete"); } @@ -108,18 +108,7 @@ public class RDBMSTransaction extends QBackendTransaction /******************************************************************************* - * Closes this stream and releases any system resources associated - * with it. If the stream is already closed then invoking this - * method has no effect. - * - *

As noted in {@link AutoCloseable#close()}, cases where the - * close may fail require careful attention. It is strongly advised - * to relinquish the underlying resources and to internally - * mark the {@code Closeable} as closed, prior to throwing - * the {@code IOException}. - * - * @throws IOException - * if an I/O error occurs + ** *******************************************************************************/ @Override public void close() diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java index e5961b38..b5572324 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.rdbms.jdbc; import java.io.Serializable; import java.math.BigDecimal; +import java.math.BigInteger; import java.sql.Connection; import java.sql.Date; import java.sql.PreparedStatement; @@ -188,8 +189,6 @@ public class QueryManager @SuppressWarnings("unchecked") public static T executeStatementForSingleValue(Connection connection, Class returnClass, String sql, Object... params) throws SQLException { - throw (new NotImplementedException()); - /* PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); statement.execute(); ResultSet resultSet = statement.getResultSet(); @@ -238,7 +237,6 @@ public class QueryManager { return (null); } - */ } @@ -1397,6 +1395,38 @@ public class QueryManager + /******************************************************************************* + ** + *******************************************************************************/ + public static Instant getInstant(ResultSet resultSet, String column) throws SQLException + { + Timestamp value = resultSet.getTimestamp(column); + if(resultSet.wasNull()) + { + return (null); + } + + return (value.toInstant()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Instant getInstant(ResultSet resultSet, int column) throws SQLException + { + Timestamp value = resultSet.getTimestamp(column); + if(resultSet.wasNull()) + { + return (null); + } + + return (value.toInstant()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java index 96b75742..18b38403 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java @@ -22,14 +22,22 @@ package com.kingsrook.qqq.backend.module.rdbms; +import java.io.InputStream; +import java.sql.Connection; +import java.util.List; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSActionTest; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import org.apache.commons.io.IOUtils; +import static junit.framework.Assert.assertNotNull; /******************************************************************************* @@ -42,6 +50,29 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static void primeTestDatabase(String sqlFileName) throws Exception + { + ConnectionManager connectionManager = new ConnectionManager(); + try(Connection connection = connectionManager.getConnection(TestUtils.defineBackend())) + { + InputStream primeTestDatabaseSqlStream = RDBMSActionTest.class.getResourceAsStream("/" + sqlFileName); + assertNotNull(primeTestDatabaseSqlStream); + List lines = (List) IOUtils.readLines(primeTestDatabaseSqlStream); + lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList(); + String joinedSQL = String.join("\n", lines); + for(String sql : joinedSQL.split(";")) + { + QueryManager.executeUpdate(connection, sql); + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java index 7382de50..398f6f5f 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java @@ -22,15 +22,11 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; -import java.io.InputStream; import java.sql.Connection; -import java.util.List; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; -import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.AfterEach; -import static junit.framework.Assert.assertNotNull; /******************************************************************************* @@ -57,33 +53,11 @@ public class RDBMSActionTest *******************************************************************************/ protected void primeTestDatabase() throws Exception { - primeTestDatabase("prime-test-database.sql"); + TestUtils.primeTestDatabase("prime-test-database.sql"); } - /******************************************************************************* - ** - *******************************************************************************/ - @SuppressWarnings("unchecked") - protected void primeTestDatabase(String sqlFileName) throws Exception - { - ConnectionManager connectionManager = new ConnectionManager(); - try(Connection connection = connectionManager.getConnection(TestUtils.defineBackend())) - { - InputStream primeTestDatabaseSqlStream = RDBMSActionTest.class.getResourceAsStream("/" + sqlFileName); - assertNotNull(primeTestDatabaseSqlStream); - List lines = (List) IOUtils.readLines(primeTestDatabaseSqlStream); - lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList(); - String joinedSQL = String.join("\n", lines); - for(String sql : joinedSQL.split(";")) - { - QueryManager.executeUpdate(connection, sql); - } - } - } - - /******************************************************************************* ** diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteActionTest.java index 0bd18c51..d006bf4c 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteActionTest.java @@ -151,7 +151,7 @@ public class RDBMSDeleteActionTest extends RDBMSActionTest ////////////////////////////////////////////////////////////////// // load the parent-child tables, with foreign keys and instance // ////////////////////////////////////////////////////////////////// - super.primeTestDatabase("prime-test-database-parent-child-tables.sql"); + TestUtils.primeTestDatabase("prime-test-database-parent-child-tables.sql"); DeleteInput deleteInput = initChildTableInstanceAndDeleteRequest(); ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransactionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransactionTest.java new file mode 100644 index 00000000..7357cb4c --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSTransactionTest.java @@ -0,0 +1,96 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.actions; + + +import java.sql.Connection; +import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for RDBMSTransaction + *******************************************************************************/ +class RDBMSTransactionTest +{ + private final String testToken = getClass().getName(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + protected void beforeEach() throws Exception + { + TestUtils.primeTestDatabase("prime-test-database.sql"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCommit() throws Exception + { + ConnectionManager connectionManager = new ConnectionManager(); + Connection connection = connectionManager.getConnection(TestUtils.defineBackend()); + Integer preCount = QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT COUNT(*) FROM person"); + + Connection connectionForTransaction = connectionManager.getConnection(TestUtils.defineBackend()); + RDBMSTransaction transaction = new RDBMSTransaction(connectionForTransaction); + + QueryManager.executeUpdate(transaction.getConnection(), "INSERT INTO person (first_name, last_name, email) VALUES (?, ?, ?)", testToken, testToken, testToken); + transaction.commit(); + + Integer postCount = QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT COUNT(*) FROM person"); + assertEquals(preCount + 1, postCount); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRollback() throws Exception + { + ConnectionManager connectionManager = new ConnectionManager(); + Connection connection = connectionManager.getConnection(TestUtils.defineBackend()); + Integer preCount = QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT COUNT(*) FROM person"); + + Connection connectionForTransaction = connectionManager.getConnection(TestUtils.defineBackend()); + RDBMSTransaction transaction = new RDBMSTransaction(connectionForTransaction); + + QueryManager.executeUpdate(transaction.getConnection(), "INSERT INTO person (first_name, last_name, email) VALUES (?, ?, ?)", testToken, testToken, testToken); + transaction.rollback(); + + Integer postCount = QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT COUNT(*) FROM person"); + assertEquals(preCount, postCount); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java index e62ac2ae..ea5e99cc 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java @@ -329,4 +329,35 @@ class QueryManagerTest assertEquals(59, localTime.getSecond(), "Second value"); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testExecuteStatementForSingleValue() throws SQLException + { + Connection connection = getConnection(); + QueryManager.executeUpdate(connection, """ + INSERT INTO test_table + ( int_col, datetime_col, char_col, date_col, time_col ) + VALUES + ( 47, '2022-08-10 19:22:08', 'Q', '2022-08-10', '19:22:08') + """); + assertEquals(null, QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT int_col FROM test_table WHERE int_col = -1")); + assertEquals(1, QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT COUNT(*) FROM test_table")); + assertEquals(47, QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT int_col FROM test_table")); + assertEquals("Q", QueryManager.executeStatementForSingleValue(connection, String.class, "SELECT char_col FROM test_table")); + assertEquals(new BigDecimal("1.1"), QueryManager.executeStatementForSingleValue(connection, BigDecimal.class, "SELECT 1.1 FROM test_table")); + assertEquals(1, QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT 1.1 FROM test_table")); + + QueryManager.executeUpdate(connection, """ + INSERT INTO test_table + ( int_col, datetime_col, char_col, date_col, time_col ) + VALUES + ( null, null, null, null, null) + """); + assertEquals(null, QueryManager.executeStatementForSingleValue(connection, Integer.class, "SELECT int_col FROM test_table WHERE int_col IS NULL")); + } + } \ No newline at end of file diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index ed82ff44..2c9bb956 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -26,6 +26,7 @@ import java.util.List; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QValueException; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; @@ -227,7 +228,7 @@ public class SampleMetaDataProvider table.addField(new QFieldMetaData("company_code", QFieldType.STRING) // todo PVS .withLabel("Company") .withIsRequired(true) - .withBackendName("comp_code")); + .withBackendName("company_code")); table.addField(new QFieldMetaData("service_level", QFieldType.STRING) // todo PVS .withLabel("Service Level") @@ -246,7 +247,7 @@ public class SampleMetaDataProvider *******************************************************************************/ public static QTableMetaData defineTablePerson() { - return new QTableMetaData() + QTableMetaData qTableMetaData = new QTableMetaData() .withName(TABLE_NAME_PERSON) .withLabel("Person") .withBackendName(RDBMS_BACKEND_NAME) @@ -266,9 +267,11 @@ public class SampleMetaDataProvider .withSection(new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, List.of("id", "firstName", "lastName"))) .withSection(new QFieldSection("basicInfo", "Basic Info", new QIcon("dataset"), Tier.T2, List.of("email", "birthDate"))) .withSection(new QFieldSection("employmentInfo", "Employment Info", new QIcon("work"), Tier.T2, List.of("annualSalary", "daysWorked"))) - .withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))) + .withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); - .withInferredFieldBackendNames(); + QInstanceEnricher.setInferredFieldBackendNames(qTableMetaData); + + return (qTableMetaData); }