diff --git a/pom.xml b/pom.xml index 2198eda5..aa9a813e 100644 --- a/pom.xml +++ b/pom.xml @@ -71,6 +71,12 @@ commons-csv 1.8 + + org.assertj + assertj-core + 3.23.1 + test + diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/InsertAction.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/InsertAction.java index 888712d7..3bc0b5c8 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/actions/InsertAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/InsertAction.java @@ -27,6 +27,8 @@ import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -35,6 +37,10 @@ import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface *******************************************************************************/ public class InsertAction { + private static final Logger LOG = LogManager.getLogger(InsertAction.class); + + + /******************************************************************************* ** *******************************************************************************/ @@ -43,10 +49,11 @@ public class InsertAction ActionHelper.validateSession(insertRequest); QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); - QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(insertRequest.getBackend()); + QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(insertRequest.getBackend()); // todo pre-customization - just get to modify the request? InsertResult insertResult = qModule.getInsertInterface().execute(insertRequest); // todo post-customization - can do whatever w/ the result if you want return insertResult; } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/RunFunctionAction.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/RunFunctionAction.java index 326d8f7a..58ce72c0 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/actions/RunFunctionAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/RunFunctionAction.java @@ -39,6 +39,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -47,6 +49,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; *******************************************************************************/ public class RunFunctionAction { + private static final Logger LOG = LogManager.getLogger(RunFunctionAction.class); /******************************************************************************* ** @@ -100,8 +103,15 @@ public class RunFunctionAction Serializable value = runFunctionRequest.getValue(field.getName()); if(value == null) { - // todo - check if required? - fieldsToGet.add(field); + if(field.getDefaultValue() != null) + { + runFunctionRequest.addValue(field.getName(), field.getDefaultValue()); + } + else + { + // todo - check if required? + fieldsToGet.add(field); + } } } @@ -183,7 +193,7 @@ public class RunFunctionAction { runFunctionResult = new RunFunctionResult(); runFunctionResult.setError("Error running function code: " + e.getMessage()); - e.printStackTrace(); + LOG.info("Error running function code", e); } return (runFunctionResult); diff --git a/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java b/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java index b7eefddb..7efbecff 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java @@ -73,6 +73,8 @@ public class CsvToQRecordAdapter .withTrim()); List headers = csvParser.getHeaderNames(); + headers = makeHeadersUnique(headers); + List csvRecords = csvParser.getRecords(); for(CSVRecord csvRecord : csvRecords) { @@ -80,9 +82,9 @@ public class CsvToQRecordAdapter // put values from the CSV record into a map of header -> value // ////////////////////////////////////////////////////////////////// Map csvValues = new HashMap<>(); - for(String header : headers) + for(int i=0; i value // ///////////////////////////////////////////////////////////////// Map csvValues = new HashMap<>(); - int index = 1; + int index = 1; for(String value : csvRecord) { csvValues.put(index++, value); @@ -144,4 +146,41 @@ public class CsvToQRecordAdapter return (rs); } + + + /******************************************************************************* + ** For a list of headers, if any duplicates are found, add a numeric suffix + ** to the duplicates. + ** + ** So this header row: A,B,C,C,C + ** Would become: A,B,C,C 2,C 3 + ** + ** See unit test for more scenarios - some of which we do not handle well yet, + ** such as "C 2, C, C 3" + *******************************************************************************/ + protected List makeHeadersUnique(List headers) + { + Map countsByHeader = new HashMap<>(); + List rs = new ArrayList<>(); + + for(String header : headers) + { + String headerToUse = header; + String headerWithoutSuffix = header.replaceFirst(" \\d+$", ""); + + if(countsByHeader.containsKey(headerWithoutSuffix)) + { + int suffix = countsByHeader.get(headerWithoutSuffix) + 1; + countsByHeader.put(headerWithoutSuffix, suffix); + headerToUse = headerWithoutSuffix + " " + suffix; + } + else + { + countsByHeader.put(headerWithoutSuffix, 1); + } + rs.add(headerToUse); + } + return (rs); + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapter.java b/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapter.java index 06b4eabe..e2585efe 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapter.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapter.java @@ -55,6 +55,7 @@ public class JsonToQFieldMappingAdapter try { JSONObject jsonObject = JsonUtils.toJSONObject(json); + jsonObject = promoteInnerMappingIfAppropriate(jsonObject); ////////////////////////////////////////////////////////////////////////////////////////////// // look at the keys in the mapping - if they're strings, then we're doing key-based mapping // @@ -101,6 +102,29 @@ public class JsonToQFieldMappingAdapter + /******************************************************************************* + ** So - this class was first written assuming that the JSON it would take would + ** just be a mapping - e.g., {a:b, c:d} or {a:0, b:1}. + ** + ** But - it turns out, callers may expect that they can create an instance of + ** AbstractQFieldMapping, then serialize it, then de-serialize it, and that seems sane. + ** + ** So - this method tries to determine if the JSON Object we took in looks like + ** a serialized from of a AbstractQFieldMapping - and if so, then it "promotes" + ** the "mapping" object from within that outer json object, since the rest of + ** this class knows how to (and expects to) handle that object. + *******************************************************************************/ + private JSONObject promoteInnerMappingIfAppropriate(JSONObject jsonObject) + { + if(jsonObject.has("mapping") && jsonObject.has("sourceType") && jsonObject.keySet().size() == 2) + { + return (jsonObject.getJSONObject("mapping")); + } + return (jsonObject); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 12b74f4f..3cc63e4c 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.instances; import java.util.Locale; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; @@ -33,7 +34,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* ** As part of helping a QInstance be created and/or validated, apply some default - ** transfomations to it, such as populating missing labels based on names. + ** transformations to it, such as populating missing labels based on names. ** *******************************************************************************/ public class QInstanceEnricher @@ -52,6 +53,21 @@ public class QInstanceEnricher { qInstance.getProcesses().values().forEach(this::enrich); } + + if(qInstance.getBackends() != null) + { + qInstance.getBackends().values().forEach(this::enrich); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void enrich(QBackendMetaData qBackendMetaData) + { + qBackendMetaData.enrich(); } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 0fe65ec5..f1e018ff 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -67,7 +67,6 @@ public class QInstanceValidator } catch(Exception e) { - e.printStackTrace(); throw (new QInstanceValidationException("Error enriching qInstance prior to validation.", e)); } @@ -134,7 +133,6 @@ public class QInstanceValidator } catch(Exception e) { - e.printStackTrace(); throw (new QInstanceValidationException("Error performing qInstance validation.", e)); } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQRequest.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQRequest.java index d04523c7..9044c97a 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQRequest.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractQRequest.java @@ -27,6 +27,8 @@ import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.session.QSession; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -35,6 +37,8 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; *******************************************************************************/ public abstract class AbstractQRequest { + private static final Logger LOG = LogManager.getLogger(AbstractQRequest.class); + protected QInstance instance; protected QSession session; @@ -68,7 +72,7 @@ public abstract class AbstractQRequest } catch(QInstanceValidationException e) { - System.err.println(e.getMessage()); + LOG.warn(e); throw (new IllegalArgumentException("QInstance failed validation" + e.getMessage())); } } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunFunctionResult.java b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunFunctionResult.java index 733f947b..5d4653c2 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunFunctionResult.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunFunctionResult.java @@ -40,6 +40,20 @@ public class RunFunctionResult extends AbstractQResult + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return "RunFunctionResult{error='" + error + + ",records.size()=" + (processState == null ? null : processState.getRecords().size()) + + ",values=" + (processState == null ? null : processState.getValues()) + + "}"; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java index 24d0721e..77bc67cb 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java @@ -146,4 +146,16 @@ public class QBackendMetaData return (this); } + + + /******************************************************************************* + ** Called by the QInstanceEnricher - to do backend-type-specific enrichments. + ** Original use case is: reading secrets into fields (e.g., passwords). + *******************************************************************************/ + public void enrich() + { + //////////////////////// + // noop in base class // + //////////////////////// + } } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QCodeUsage.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QCodeUsage.java index 19cc76f7..66ab9af1 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QCodeUsage.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QCodeUsage.java @@ -28,5 +28,6 @@ package com.kingsrook.qqq.backend.core.model.metadata; *******************************************************************************/ public enum QCodeUsage { - FUNCTION + FUNCTION, // a step in a process + CUSTOMIZER // a function to customize part of a QQQ table's behavior } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QFieldMetaData.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QFieldMetaData.java index baf9473e..89d7f326 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QFieldMetaData.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QFieldMetaData.java @@ -22,6 +22,9 @@ package com.kingsrook.qqq.backend.core.model.metadata; +import java.io.Serializable; + + /******************************************************************************* ** Meta-data to represent a single field in a table. ** @@ -33,6 +36,7 @@ public class QFieldMetaData private String backendName; private QFieldType type; + private Serializable defaultValue; private String possibleValueSourceName; @@ -216,4 +220,37 @@ public class QFieldMetaData return (this); } + /******************************************************************************* + ** Getter for defaultValue + ** + *******************************************************************************/ + public Serializable getDefaultValue() + { + return defaultValue; + } + + + + /******************************************************************************* + ** Setter for defaultValue + ** + *******************************************************************************/ + public void setDefaultValue(Serializable defaultValue) + { + this.defaultValue = defaultValue; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QFieldMetaData withDefaultValue(Serializable defaultValue) + { + this.defaultValue = defaultValue; + return (this); + } + + + } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QSecretReader.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QSecretReader.java new file mode 100644 index 00000000..4b4ea3bc --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QSecretReader.java @@ -0,0 +1,89 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata; + + +import java.util.Map; + + +/******************************************************************************* + ** To avoid having secrets (passwords, access keys, etc) committed into meta data + ** files, this class is used by the Enricher to "promote" values, such as ${env.ACCESS_KEY} + ** to be read from the environment (or other secret providers (to be implemented)). + *******************************************************************************/ +public class QSecretReader +{ + private Map customEnvironment; + + + + /******************************************************************************* + ** Translate a secret. + ** + ** If input is null, output is null. + ** If input looks like ${env.X}, then the return value is the value of the env variable 'X' + ** Else the output is the input. + *******************************************************************************/ + public String readSecret(String value) + { + if(value == null) + { + return (null); + } + + if(value.startsWith("${env.") && value.endsWith("}")) + { + String envVarName = value.substring(6).replaceFirst("}$", ""); + String envValue = getEnvironment().get(envVarName); + return (envValue); + } + + return (value); + } + + + + /******************************************************************************* + ** Setter for customEnvironment - protected - meant to be called (at least at this + ** time), only in unit test + ** + *******************************************************************************/ + protected void setCustomEnvironment(Map customEnvironment) + { + this.customEnvironment = customEnvironment; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Map getEnvironment() + { + if(this.customEnvironment != null) + { + return (this.customEnvironment); + } + + return System.getenv(); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QTableMetaData.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QTableMetaData.java index f479e463..dfe40a4f 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QTableMetaData.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QTableMetaData.java @@ -23,8 +23,11 @@ package com.kingsrook.qqq.backend.core.model.metadata; import java.io.Serializable; +import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.Optional; /******************************************************************************* @@ -51,6 +54,8 @@ public class QTableMetaData implements Serializable private QTableBackendDetails backendDetails; + private Map customizers; + /******************************************************************************* @@ -241,6 +246,21 @@ public class QTableMetaData implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData withFields(List fields) + { + this.fields = new LinkedHashMap<>(); + for(QFieldMetaData field : fields) + { + this.addField(field); + } + return (this); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -250,6 +270,12 @@ public class QTableMetaData implements Serializable { this.fields = new LinkedHashMap<>(); } + + if(this.fields.containsKey(field.getName())) + { + throw (new IllegalArgumentException("Attempt to add a second field with name [" + field.getName() + "] to table [" + name + "].")); + } + this.fields.put(field.getName(), field); } @@ -291,6 +317,7 @@ public class QTableMetaData implements Serializable } + /******************************************************************************* ** Fluent Setter for backendDetails ** @@ -301,4 +328,72 @@ public class QTableMetaData implements Serializable return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public Optional getCustomizer(String customizerName) + { + if(customizers == null) + { + return (Optional.empty()); + } + + QCodeReference function = customizers.get(customizerName); + if(function == null) + { + throw (new IllegalArgumentException("Customizer [" + customizerName + "] was not found in table [" + name + "].")); + } + + return (Optional.of(function)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Map getCustomizers() + { + return customizers; + } + + + + /******************************************************************************* + ** Setter for customizers + ** + *******************************************************************************/ + public void setCustomizers(Map customizers) + { + this.customizers = customizers; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData withCustomizer(String role, QCodeReference customizer) + { + if(this.customizers == null) + { + this.customizers = new HashMap<>(); + } + // todo - check for dupes? + this.customizers.put(role, customizer); + return (this); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData withCustomizers(Map customizers) + { + this.customizers = customizers; + return (this); + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionInputMetaData.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionInputMetaData.java index fdf8ed31..a0221b93 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionInputMetaData.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionInputMetaData.java @@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes; import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; @@ -33,7 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; *******************************************************************************/ public class QFunctionInputMetaData { - private QRecordListMetaData recordListMetaData; + private QRecordListMetaData recordListMetaData; private List fieldList = new ArrayList<>(); @@ -72,6 +74,33 @@ public class QFunctionInputMetaData + /******************************************************************************* + ** Getter a field with the given name + ** + *******************************************************************************/ + public Optional getField(String name) + { + return (fieldList.stream().filter(field -> name.equals(field.getName())).findFirst()); + } + + + + /******************************************************************************* + ** Getter a field with the given name - throwing if it wasn't found + ** + *******************************************************************************/ + public QFieldMetaData getFieldThrowing(String name) throws QException + { + Optional field = fieldList.stream().filter(f -> name.equals(f.getName())).findFirst(); + if(field.isEmpty()) + { + throw (new QException("Could not find field [" + name + "] in function input meta data")); + } + return (field.get()); + } + + + /******************************************************************************* ** Getter for fieldList ** diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/QFieldMappingDeserializer.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/QFieldMappingDeserializer.java new file mode 100644 index 00000000..f093ad41 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/QFieldMappingDeserializer.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.serialization; + + +import java.io.IOException; +import com.fasterxml.jackson.core.JacksonException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.TreeNode; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping; +import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; + + +/******************************************************************************* + ** Jackson custom deserialization class, to return an appropriate sub-type of + ** A QBackendMetaData, based on the backendType specified within. + *******************************************************************************/ +public class QFieldMappingDeserializer extends JsonDeserializer +{ + @Override + public AbstractQFieldMapping deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JacksonException + { + TreeNode treeNode = jsonParser.readValueAsTree(); + + TreeNode sourceTypeTreeNode = treeNode.get("sourceType"); + if(sourceTypeTreeNode == null || sourceTypeTreeNode instanceof NullNode) + { + throw new IOException("Missing sourceType in serializedMapping"); + } + + if(!(sourceTypeTreeNode instanceof TextNode textNode)) + { + throw new IOException("sourceType is not a string value (is: " + sourceTypeTreeNode.getClass().getSimpleName() + ")"); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // get the value of the backendType json node, and use it to look up the qBackendModule object // + ///////////////////////////////////////////////////////////////////////////////////////////////// + String backendType = textNode.asText(); + + QBackendModuleInterface backendModule = DeserializerUtils.getBackendModule(treeNode); + + return null; + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/modules/QBackendModuleDispatcher.java b/src/main/java/com/kingsrook/qqq/backend/core/modules/QBackendModuleDispatcher.java index 441e5959..b01b793d 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/modules/QBackendModuleDispatcher.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/modules/QBackendModuleDispatcher.java @@ -42,7 +42,7 @@ public class QBackendModuleDispatcher { private static final Logger LOG = LogManager.getLogger(QBackendModuleDispatcher.class); - private Map backendTypeToModuleClassNameMap; + private static Map backendTypeToModuleClassNameMap; @@ -51,6 +51,21 @@ public class QBackendModuleDispatcher *******************************************************************************/ public QBackendModuleDispatcher() { + initBackendTypeToModuleClassNameMap(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void initBackendTypeToModuleClassNameMap() + { + if(backendTypeToModuleClassNameMap != null) + { + return; + } + backendTypeToModuleClassNameMap = new HashMap<>(); String[] moduleClassNames = new String[] @@ -80,6 +95,22 @@ public class QBackendModuleDispatcher + /******************************************************************************* + ** + *******************************************************************************/ + public static void registerBackendModule(QBackendModuleInterface moduleInstance) + { + initBackendTypeToModuleClassNameMap(); + String backendType = moduleInstance.getBackendType(); + if(backendTypeToModuleClassNameMap.containsKey(backendType)) + { + LOG.info("Overwriting backend type [" + backendType + "] with [" + moduleInstance.getClass() + "]"); + } + backendTypeToModuleClassNameMap.put(backendType, moduleInstance.getClass().getName()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/src/main/java/com/kingsrook/qqq/backend/core/modules/mock/MockQueryAction.java b/src/main/java/com/kingsrook/qqq/backend/core/modules/mock/MockQueryAction.java index b9475010..3b6221cd 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/modules/mock/MockQueryAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/modules/mock/MockQueryAction.java @@ -71,7 +71,6 @@ public class MockQueryAction implements QueryInterface } catch(Exception e) { - e.printStackTrace(); throw new QException("Error executing query", e); } } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java index c06c764e..b245252c 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java @@ -39,8 +39,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; public class BasicETLProcess { public static final String PROCESS_NAME = "etl.basic"; + public static final String FUNCTION_NAME_EXTRACT = "extract"; + public static final String FUNCTION_NAME_TRANSFORM = "transform"; + public static final String FUNCTION_NAME_LOAD = "load"; public static final String FIELD_SOURCE_TABLE = "sourceTable"; public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + public static final String FIELD_MAPPING_JSON = "mappingJSON"; public static final String FIELD_RECORD_COUNT = "recordCount"; @@ -51,7 +55,7 @@ public class BasicETLProcess public QProcessMetaData defineProcessMetaData() { QFunctionMetaData extractFunction = new QFunctionMetaData() - .withName("extract") + .withName(FUNCTION_NAME_EXTRACT) .withCode(new QCodeReference() .withName(BasicETLExtractFunction.class.getName()) .withCodeType(QCodeType.JAVA) @@ -59,8 +63,18 @@ public class BasicETLProcess .withInputData(new QFunctionInputMetaData() .addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING))); + QFunctionMetaData transformFunction = new QFunctionMetaData() + .withName(FUNCTION_NAME_TRANSFORM) + .withCode(new QCodeReference() + .withName(BasicETLTransformFunction.class.getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.FUNCTION)) + .withInputData(new QFunctionInputMetaData() + .addField(new QFieldMetaData(FIELD_MAPPING_JSON, QFieldType.STRING)) + .addField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING))); + QFunctionMetaData loadFunction = new QFunctionMetaData() - .withName("load") + .withName(FUNCTION_NAME_LOAD) .withCode(new QCodeReference() .withName(BasicETLLoadFunction.class.getName()) .withCodeType(QCodeType.JAVA) @@ -73,6 +87,7 @@ public class BasicETLProcess return new QProcessMetaData() .withName(PROCESS_NAME) .addFunction(extractFunction) + .addFunction(transformFunction) .addFunction(loadFunction); } } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunction.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunction.java new file mode 100644 index 00000000..9783e638 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunction.java @@ -0,0 +1,99 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.adapters.JsonToQFieldMappingAdapter; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping; +import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** Function body for performing the Extract step of a basic ETL process. + *******************************************************************************/ +public class BasicETLTransformFunction implements FunctionBody +{ + @Override + public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + { + //////////////////////////////////////////////////////////////////////////////////////////// + // exit early with no-op if no records made it here, or if we don't have a mapping to use // + //////////////////////////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeIsEmpty(runFunctionRequest.getRecords())) + { + return; + } + + String mappingJSON = runFunctionRequest.getValueString(BasicETLProcess.FIELD_MAPPING_JSON); + if(!StringUtils.hasContent(mappingJSON)) + { + return; + } + + ///////////////////////////////////////////////////////////////////////////////////////////// + // require that the mapping be a key-based mapping (can't use indexes into qRecord values) // + ///////////////////////////////////////////////////////////////////////////////////////////// + AbstractQFieldMapping mapping = new JsonToQFieldMappingAdapter().buildMappingFromJson(mappingJSON); + if(!(mapping instanceof QKeyBasedFieldMapping keyBasedFieldMapping)) + { + throw (new QException("Mapping was not a Key-based mapping type. Was a : " + mapping.getClass().getName())); + } + + String tableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE); + QTableMetaData table = runFunctionRequest.getInstance().getTable(tableName); + List mappedRecords = applyMapping(runFunctionRequest.getRecords(), table, keyBasedFieldMapping); + runFunctionResult.setRecords(mappedRecords); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List applyMapping(List input, QTableMetaData table, QKeyBasedFieldMapping mapping) + { + List output = new ArrayList<>(); + for(QRecord inputRecord : input) + { + QRecord outputRecord = new QRecord(); + output.add(outputRecord); + for(QFieldMetaData field : table.getFields().values()) + { + String fieldSource = mapping.getFieldSource(field.getName()); + outputRecord.setValue(field.getName(), inputRecord.getValue(fieldSource)); + } + } + return (output); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java b/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java index 19ea144d..9b0d4390 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java @@ -29,6 +29,8 @@ import java.io.Serializable; import java.util.Optional; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -36,6 +38,8 @@ import org.apache.commons.io.FileUtils; *******************************************************************************/ public class TempFileStateProvider implements StateProviderInterface { + private static final Logger LOG = LogManager.getLogger(TempFileStateProvider.class); + private static TempFileStateProvider instance; @@ -76,8 +80,8 @@ public class TempFileStateProvider implements StateProviderInterface } catch(IOException e) { - // todo better - e.printStackTrace(); + LOG.error("Error putting state into file", e); + throw (new RuntimeException("Error storing state", e)); } } @@ -98,9 +102,10 @@ public class TempFileStateProvider implements StateProviderInterface { return (Optional.empty()); } - catch(IOException ie) + catch(IOException e) { - throw new RuntimeException("Error loading state from file", ie); + LOG.error("Error getting state from file", e); + throw (new RuntimeException("Error retreiving state", e)); } } diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml new file mode 100644 index 00000000..db3b9781 --- /dev/null +++ b/src/main/resources/log4j2.xml @@ -0,0 +1,29 @@ + + + + %highlight{%d{ISO8601} | %-6r | %-5p | %markerSimpleName | %c{1} | %t{1} | %m%n}{FATAL=red, ERROR=red, WARN=blue, INFO=green, METRICS=magenta, DEBUG=cyan, TRACE=white} + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java b/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java index c17eb46c..2593b3e2 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java @@ -27,6 +27,7 @@ import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFi import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -230,4 +231,54 @@ class CsvToQRecordAdapterTest assertEquals("1981-01-01", qRecord2.getValue("birthDate")); } + + + /******************************************************************************* + ** In this test - we've got CSV data with duplicated header names. + ** In our mapping, we're seeing the suffixes of " 2" and " 3" addd to those + ** header names on the RHS. + *******************************************************************************/ + @Test + public void test_duplicatedColumnHeaders() + { + QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping() + .withMapping("id", "id") + .withMapping("createDate", "date") + .withMapping("modifyDate", "date 2") + .withMapping("firstName", "name") + .withMapping("lastName", "name 2") + .withMapping("birthDate", "date 3") + .withMapping("email", "email"); + + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(""" + id,date,date,name,name,date,email + 1,2022-06-26,2022-06-26,John,Doe,1980-01-01,john@kingsrook.com + """, TestUtils.defineTablePerson(), mapping); + assertNotNull(qRecords); + assertEquals(1, qRecords.size()); + QRecord qRecord1 = qRecords.get(0); + assertEquals("John", qRecord1.getValue("firstName")); + assertEquals("Doe", qRecord1.getValue("lastName")); + assertEquals("1980-01-01", qRecord1.getValue("birthDate")); + assertEquals("2022-06-26", qRecord1.getValue("createDate")); + assertEquals("2022-06-26", qRecord1.getValue("modifyDate")); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testMakeHeadersUnique() + { + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + Assertions.assertEquals(List.of("A", "B", "C"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C"))); + Assertions.assertEquals(List.of("A", "B", "C", "C 2", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C", "C", "C"))); + Assertions.assertEquals(List.of("C", "A", "C 2", "B", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("C", "A", "C", "B", "C"))); + Assertions.assertEquals(List.of("A", "B", "C", "C 2", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C", "C 2", "C"))); + Assertions.assertEquals(List.of("A", "B", "C", "C 2", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C", "C 2", "C 3"))); + // todo - this is what the method header comment means when it says we don't handle all cases well... + // Assertions.assertEquals(List.of("A", "B", "C", "C 2", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C 2", "C", "C 3"))); + } } diff --git a/src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapterTest.java b/src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapterTest.java index 86454a7c..c7146f3a 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapterTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQFieldMappingAdapterTest.java @@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.adapters; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFieldMapping; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -130,6 +132,38 @@ class JsonToQFieldMappingAdapterTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_deserializeSerializedQKeyBasedFieldMapping() + { + QIndexBasedFieldMapping original = new QIndexBasedFieldMapping() + .withMapping("foo", 0) + .withMapping("bar", 1); + String json = JsonUtils.toJson(original); + AbstractQFieldMapping deserialized = new JsonToQFieldMappingAdapter().buildMappingFromJson(json); + Assertions.assertThat(deserialized).usingRecursiveComparison().isEqualTo(original); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_deserializeSerializedQIndexBasedFieldMapping() + { + QKeyBasedFieldMapping original = new QKeyBasedFieldMapping() + .withMapping("foo", "Fu") + .withMapping("bar", "Bahr"); + String json = JsonUtils.toJson(original); + AbstractQFieldMapping deserialized = new JsonToQFieldMappingAdapter().buildMappingFromJson(json); + Assertions.assertThat(deserialized).usingRecursiveComparison().isEqualTo(original); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -140,6 +174,7 @@ class JsonToQFieldMappingAdapterTest } + /******************************************************************************* ** *******************************************************************************/ @@ -159,7 +194,7 @@ class JsonToQFieldMappingAdapterTest try { JsonToQFieldMappingAdapter jsonToQFieldMappingAdapter = new JsonToQFieldMappingAdapter(); - AbstractQFieldMapping mapping = jsonToQFieldMappingAdapter.buildMappingFromJson(json); + AbstractQFieldMapping mapping = jsonToQFieldMappingAdapter.buildMappingFromJson(json); System.out.println(mapping); } catch(IllegalArgumentException iae) diff --git a/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/QSecretReaderTest.java b/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/QSecretReaderTest.java new file mode 100644 index 00000000..91124829 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/QSecretReaderTest.java @@ -0,0 +1,55 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata; + + +import java.util.Map; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for QSecretReader + *******************************************************************************/ +class QSecretReaderTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testReadSecret() + { + QSecretReader secretReader = new QSecretReader(); + String key = "CUSTOM_PROPERTY"; + String value = "ABCD-9876"; + secretReader.setCustomEnvironment(Map.of(key, value)); + + assertNull(secretReader.readSecret(null)); + assertEquals("foo", secretReader.readSecret("foo")); + assertNull(secretReader.readSecret("${env.NOT-" + key + "}")); + assertEquals(value, secretReader.readSecret("${env." + key + "}")); + assertEquals("${env.NOT-" + key, secretReader.readSecret("${env.NOT-" + key)); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java index 84eb7d83..40cde032 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java @@ -26,6 +26,8 @@ import com.kingsrook.qqq.backend.core.actions.RunProcessAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessRequest; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessResult; +import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -40,7 +42,7 @@ class BasicETLProcessTest { /******************************************************************************* - ** + ** Simplest happy path *******************************************************************************/ @Test public void test() throws QException @@ -50,6 +52,7 @@ class BasicETLProcessTest request.setProcessName(BasicETLProcess.PROCESS_NAME); request.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.defineTablePerson().getName()); request.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.definePersonFileTable().getName()); + request.addValue(BasicETLProcess.FIELD_MAPPING_JSON, ""); RunProcessResult result = new RunProcessAction().execute(request); assertNotNull(result); @@ -58,4 +61,30 @@ class BasicETLProcessTest } + + /******************************************************************************* + ** Basic example of doing a mapping transformation + *******************************************************************************/ + @Test + public void testMappingTransformation() throws QException + { + RunProcessRequest request = new RunProcessRequest(TestUtils.defineInstance()); + request.setSession(TestUtils.getMockSession()); + request.setProcessName(BasicETLProcess.PROCESS_NAME); + request.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.definePersonFileTable().getName()); + request.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.defineTableIdAndNameOnly().getName()); + + /////////////////////////////////////////////////////////////////////////////////////// + // define our mapping from destination-table field names to source-table field names // + /////////////////////////////////////////////////////////////////////////////////////// + QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping().withMapping("name", "firstName"); + // request.addValue(BasicETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping.getMapping())); + request.addValue(BasicETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping)); + + RunProcessResult result = new RunProcessAction().execute(request); + assertNotNull(result); + assertNull(result.getError()); + assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process"); + } + } \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 92862ea8..e8a70e60 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -68,6 +68,7 @@ public class TestUtils qInstance.addBackend(defineBackend()); qInstance.addTable(defineTablePerson()); qInstance.addTable(definePersonFileTable()); + qInstance.addTable(defineTableIdAndNameOnly()); qInstance.addPossibleValueSource(defineStatesPossibleValueSource()); qInstance.addProcess(defineProcessGreetPeople()); qInstance.addProcess(defineProcessAddToPeoplesAge()); @@ -153,6 +154,22 @@ public class TestUtils + /******************************************************************************* + ** Define simple table with just an id and name + *******************************************************************************/ + public static QTableMetaData defineTableIdAndNameOnly() + { + return new QTableMetaData() + .withName("idAndNameOnly") + .withLabel("Id and Name Only") + .withBackendName(DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("name", QFieldType.STRING)); + } + + + /******************************************************************************* ** Define the 'greet people' process *******************************************************************************/