mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 05:01:07 +00:00
QQQ-14 add customizer functions; add log4j config; add SecretReader;
This commit is contained in:
6
pom.xml
6
pom.xml
@ -71,6 +71,12 @@
|
||||
<artifactId>commons-csv</artifactId>
|
||||
<version>1.8</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.assertj</groupId>
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<version>3.23.1</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- Common deps for all qqq modules -->
|
||||
<dependency>
|
||||
|
@ -27,6 +27,8 @@ import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
|
||||
import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -35,6 +37,10 @@ import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface
|
||||
*******************************************************************************/
|
||||
public class InsertAction
|
||||
{
|
||||
private static final Logger LOG = LogManager.getLogger(InsertAction.class);
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -43,10 +49,11 @@ public class InsertAction
|
||||
ActionHelper.validateSession(insertRequest);
|
||||
|
||||
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
|
||||
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(insertRequest.getBackend());
|
||||
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(insertRequest.getBackend());
|
||||
// todo pre-customization - just get to modify the request?
|
||||
InsertResult insertResult = qModule.getInsertInterface().execute(insertRequest);
|
||||
// todo post-customization - can do whatever w/ the result if you want
|
||||
return insertResult;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -39,6 +39,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -47,6 +49,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
*******************************************************************************/
|
||||
public class RunFunctionAction
|
||||
{
|
||||
private static final Logger LOG = LogManager.getLogger(RunFunctionAction.class);
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
@ -100,8 +103,15 @@ public class RunFunctionAction
|
||||
Serializable value = runFunctionRequest.getValue(field.getName());
|
||||
if(value == null)
|
||||
{
|
||||
// todo - check if required?
|
||||
fieldsToGet.add(field);
|
||||
if(field.getDefaultValue() != null)
|
||||
{
|
||||
runFunctionRequest.addValue(field.getName(), field.getDefaultValue());
|
||||
}
|
||||
else
|
||||
{
|
||||
// todo - check if required?
|
||||
fieldsToGet.add(field);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -183,7 +193,7 @@ public class RunFunctionAction
|
||||
{
|
||||
runFunctionResult = new RunFunctionResult();
|
||||
runFunctionResult.setError("Error running function code: " + e.getMessage());
|
||||
e.printStackTrace();
|
||||
LOG.info("Error running function code", e);
|
||||
}
|
||||
|
||||
return (runFunctionResult);
|
||||
|
@ -73,6 +73,8 @@ public class CsvToQRecordAdapter
|
||||
.withTrim());
|
||||
|
||||
List<String> headers = csvParser.getHeaderNames();
|
||||
headers = makeHeadersUnique(headers);
|
||||
|
||||
List<CSVRecord> csvRecords = csvParser.getRecords();
|
||||
for(CSVRecord csvRecord : csvRecords)
|
||||
{
|
||||
@ -80,9 +82,9 @@ public class CsvToQRecordAdapter
|
||||
// put values from the CSV record into a map of header -> value //
|
||||
//////////////////////////////////////////////////////////////////
|
||||
Map<String, String> csvValues = new HashMap<>();
|
||||
for(String header : headers)
|
||||
for(int i=0; i<headers.size(); i++)
|
||||
{
|
||||
csvValues.put(header, csvRecord.get(header));
|
||||
csvValues.put(headers.get(i), csvRecord.get(i));
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@ -113,7 +115,7 @@ public class CsvToQRecordAdapter
|
||||
// put values from the CSV record into a map of index -> value //
|
||||
/////////////////////////////////////////////////////////////////
|
||||
Map<Integer, String> csvValues = new HashMap<>();
|
||||
int index = 1;
|
||||
int index = 1;
|
||||
for(String value : csvRecord)
|
||||
{
|
||||
csvValues.put(index++, value);
|
||||
@ -144,4 +146,41 @@ public class CsvToQRecordAdapter
|
||||
return (rs);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** For a list of headers, if any duplicates are found, add a numeric suffix
|
||||
** to the duplicates.
|
||||
**
|
||||
** So this header row: A,B,C,C,C
|
||||
** Would become: A,B,C,C 2,C 3
|
||||
**
|
||||
** See unit test for more scenarios - some of which we do not handle well yet,
|
||||
** such as "C 2, C, C 3"
|
||||
*******************************************************************************/
|
||||
protected List<String> makeHeadersUnique(List<String> headers)
|
||||
{
|
||||
Map<String, Integer> countsByHeader = new HashMap<>();
|
||||
List<String> rs = new ArrayList<>();
|
||||
|
||||
for(String header : headers)
|
||||
{
|
||||
String headerToUse = header;
|
||||
String headerWithoutSuffix = header.replaceFirst(" \\d+$", "");
|
||||
|
||||
if(countsByHeader.containsKey(headerWithoutSuffix))
|
||||
{
|
||||
int suffix = countsByHeader.get(headerWithoutSuffix) + 1;
|
||||
countsByHeader.put(headerWithoutSuffix, suffix);
|
||||
headerToUse = headerWithoutSuffix + " " + suffix;
|
||||
}
|
||||
else
|
||||
{
|
||||
countsByHeader.put(headerWithoutSuffix, 1);
|
||||
}
|
||||
rs.add(headerToUse);
|
||||
}
|
||||
return (rs);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -55,6 +55,7 @@ public class JsonToQFieldMappingAdapter
|
||||
try
|
||||
{
|
||||
JSONObject jsonObject = JsonUtils.toJSONObject(json);
|
||||
jsonObject = promoteInnerMappingIfAppropriate(jsonObject);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// look at the keys in the mapping - if they're strings, then we're doing key-based mapping //
|
||||
@ -101,6 +102,29 @@ public class JsonToQFieldMappingAdapter
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** So - this class was first written assuming that the JSON it would take would
|
||||
** just be a mapping - e.g., {a:b, c:d} or {a:0, b:1}.
|
||||
**
|
||||
** But - it turns out, callers may expect that they can create an instance of
|
||||
** AbstractQFieldMapping, then serialize it, then de-serialize it, and that seems sane.
|
||||
**
|
||||
** So - this method tries to determine if the JSON Object we took in looks like
|
||||
** a serialized from of a AbstractQFieldMapping - and if so, then it "promotes"
|
||||
** the "mapping" object from within that outer json object, since the rest of
|
||||
** this class knows how to (and expects to) handle that object.
|
||||
*******************************************************************************/
|
||||
private JSONObject promoteInnerMappingIfAppropriate(JSONObject jsonObject)
|
||||
{
|
||||
if(jsonObject.has("mapping") && jsonObject.has("sourceType") && jsonObject.keySet().size() == 2)
|
||||
{
|
||||
return (jsonObject.getJSONObject("mapping"));
|
||||
}
|
||||
return (jsonObject);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.instances;
|
||||
|
||||
|
||||
import java.util.Locale;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
@ -33,7 +34,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
|
||||
/*******************************************************************************
|
||||
** As part of helping a QInstance be created and/or validated, apply some default
|
||||
** transfomations to it, such as populating missing labels based on names.
|
||||
** transformations to it, such as populating missing labels based on names.
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class QInstanceEnricher
|
||||
@ -52,6 +53,21 @@ public class QInstanceEnricher
|
||||
{
|
||||
qInstance.getProcesses().values().forEach(this::enrich);
|
||||
}
|
||||
|
||||
if(qInstance.getBackends() != null)
|
||||
{
|
||||
qInstance.getBackends().values().forEach(this::enrich);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void enrich(QBackendMetaData qBackendMetaData)
|
||||
{
|
||||
qBackendMetaData.enrich();
|
||||
}
|
||||
|
||||
|
||||
|
@ -67,7 +67,6 @@ public class QInstanceValidator
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
e.printStackTrace();
|
||||
throw (new QInstanceValidationException("Error enriching qInstance prior to validation.", e));
|
||||
}
|
||||
|
||||
@ -134,7 +133,6 @@ public class QInstanceValidator
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
e.printStackTrace();
|
||||
throw (new QInstanceValidationException("Error performing qInstance validation.", e));
|
||||
}
|
||||
|
||||
|
@ -27,6 +27,8 @@ import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -35,6 +37,8 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
*******************************************************************************/
|
||||
public abstract class AbstractQRequest
|
||||
{
|
||||
private static final Logger LOG = LogManager.getLogger(AbstractQRequest.class);
|
||||
|
||||
protected QInstance instance;
|
||||
protected QSession session;
|
||||
|
||||
@ -68,7 +72,7 @@ public abstract class AbstractQRequest
|
||||
}
|
||||
catch(QInstanceValidationException e)
|
||||
{
|
||||
System.err.println(e.getMessage());
|
||||
LOG.warn(e);
|
||||
throw (new IllegalArgumentException("QInstance failed validation" + e.getMessage()));
|
||||
}
|
||||
}
|
||||
|
@ -40,6 +40,20 @@ public class RunFunctionResult extends AbstractQResult
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return "RunFunctionResult{error='" + error
|
||||
+ ",records.size()=" + (processState == null ? null : processState.getRecords().size())
|
||||
+ ",values=" + (processState == null ? null : processState.getValues())
|
||||
+ "}";
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
@ -146,4 +146,16 @@ public class QBackendMetaData
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Called by the QInstanceEnricher - to do backend-type-specific enrichments.
|
||||
** Original use case is: reading secrets into fields (e.g., passwords).
|
||||
*******************************************************************************/
|
||||
public void enrich()
|
||||
{
|
||||
////////////////////////
|
||||
// noop in base class //
|
||||
////////////////////////
|
||||
}
|
||||
}
|
||||
|
@ -28,5 +28,6 @@ package com.kingsrook.qqq.backend.core.model.metadata;
|
||||
*******************************************************************************/
|
||||
public enum QCodeUsage
|
||||
{
|
||||
FUNCTION
|
||||
FUNCTION, // a step in a process
|
||||
CUSTOMIZER // a function to customize part of a QQQ table's behavior
|
||||
}
|
||||
|
@ -22,6 +22,9 @@
|
||||
package com.kingsrook.qqq.backend.core.model.metadata;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Meta-data to represent a single field in a table.
|
||||
**
|
||||
@ -33,6 +36,7 @@ public class QFieldMetaData
|
||||
private String backendName;
|
||||
private QFieldType type;
|
||||
|
||||
private Serializable defaultValue;
|
||||
private String possibleValueSourceName;
|
||||
|
||||
|
||||
@ -216,4 +220,37 @@ public class QFieldMetaData
|
||||
return (this);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for defaultValue
|
||||
**
|
||||
*******************************************************************************/
|
||||
public Serializable getDefaultValue()
|
||||
{
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for defaultValue
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void setDefaultValue(Serializable defaultValue)
|
||||
{
|
||||
this.defaultValue = defaultValue;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public QFieldMetaData withDefaultValue(Serializable defaultValue)
|
||||
{
|
||||
this.defaultValue = defaultValue;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
@ -0,0 +1,89 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.model.metadata;
|
||||
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** To avoid having secrets (passwords, access keys, etc) committed into meta data
|
||||
** files, this class is used by the Enricher to "promote" values, such as ${env.ACCESS_KEY}
|
||||
** to be read from the environment (or other secret providers (to be implemented)).
|
||||
*******************************************************************************/
|
||||
public class QSecretReader
|
||||
{
|
||||
private Map<String, String> customEnvironment;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Translate a secret.
|
||||
**
|
||||
** If input is null, output is null.
|
||||
** If input looks like ${env.X}, then the return value is the value of the env variable 'X'
|
||||
** Else the output is the input.
|
||||
*******************************************************************************/
|
||||
public String readSecret(String value)
|
||||
{
|
||||
if(value == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
if(value.startsWith("${env.") && value.endsWith("}"))
|
||||
{
|
||||
String envVarName = value.substring(6).replaceFirst("}$", "");
|
||||
String envValue = getEnvironment().get(envVarName);
|
||||
return (envValue);
|
||||
}
|
||||
|
||||
return (value);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for customEnvironment - protected - meant to be called (at least at this
|
||||
** time), only in unit test
|
||||
**
|
||||
*******************************************************************************/
|
||||
protected void setCustomEnvironment(Map<String, String> customEnvironment)
|
||||
{
|
||||
this.customEnvironment = customEnvironment;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private Map<String, String> getEnvironment()
|
||||
{
|
||||
if(this.customEnvironment != null)
|
||||
{
|
||||
return (this.customEnvironment);
|
||||
}
|
||||
|
||||
return System.getenv();
|
||||
}
|
||||
}
|
@ -23,8 +23,11 @@ package com.kingsrook.qqq.backend.core.model.metadata;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -51,6 +54,8 @@ public class QTableMetaData implements Serializable
|
||||
|
||||
private QTableBackendDetails backendDetails;
|
||||
|
||||
private Map<String, QCodeReference> customizers;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -241,6 +246,21 @@ public class QTableMetaData implements Serializable
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public QTableMetaData withFields(List<QFieldMetaData> fields)
|
||||
{
|
||||
this.fields = new LinkedHashMap<>();
|
||||
for(QFieldMetaData field : fields)
|
||||
{
|
||||
this.addField(field);
|
||||
}
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -250,6 +270,12 @@ public class QTableMetaData implements Serializable
|
||||
{
|
||||
this.fields = new LinkedHashMap<>();
|
||||
}
|
||||
|
||||
if(this.fields.containsKey(field.getName()))
|
||||
{
|
||||
throw (new IllegalArgumentException("Attempt to add a second field with name [" + field.getName() + "] to table [" + name + "]."));
|
||||
}
|
||||
|
||||
this.fields.put(field.getName(), field);
|
||||
}
|
||||
|
||||
@ -291,6 +317,7 @@ public class QTableMetaData implements Serializable
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent Setter for backendDetails
|
||||
**
|
||||
@ -301,4 +328,72 @@ public class QTableMetaData implements Serializable
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public Optional<QCodeReference> getCustomizer(String customizerName)
|
||||
{
|
||||
if(customizers == null)
|
||||
{
|
||||
return (Optional.empty());
|
||||
}
|
||||
|
||||
QCodeReference function = customizers.get(customizerName);
|
||||
if(function == null)
|
||||
{
|
||||
throw (new IllegalArgumentException("Customizer [" + customizerName + "] was not found in table [" + name + "]."));
|
||||
}
|
||||
|
||||
return (Optional.of(function));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public Map<String, QCodeReference> getCustomizers()
|
||||
{
|
||||
return customizers;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for customizers
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void setCustomizers(Map<String, QCodeReference> customizers)
|
||||
{
|
||||
this.customizers = customizers;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public QTableMetaData withCustomizer(String role, QCodeReference customizer)
|
||||
{
|
||||
if(this.customizers == null)
|
||||
{
|
||||
this.customizers = new HashMap<>();
|
||||
}
|
||||
// todo - check for dupes?
|
||||
this.customizers.put(role, customizer);
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public QTableMetaData withCustomizers(Map<String, QCodeReference> customizers)
|
||||
{
|
||||
this.customizers = customizers;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||
|
||||
|
||||
@ -33,7 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||
*******************************************************************************/
|
||||
public class QFunctionInputMetaData
|
||||
{
|
||||
private QRecordListMetaData recordListMetaData;
|
||||
private QRecordListMetaData recordListMetaData;
|
||||
private List<QFieldMetaData> fieldList = new ArrayList<>();
|
||||
|
||||
|
||||
@ -72,6 +74,33 @@ public class QFunctionInputMetaData
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter a field with the given name
|
||||
**
|
||||
*******************************************************************************/
|
||||
public Optional<QFieldMetaData> getField(String name)
|
||||
{
|
||||
return (fieldList.stream().filter(field -> name.equals(field.getName())).findFirst());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter a field with the given name - throwing if it wasn't found
|
||||
**
|
||||
*******************************************************************************/
|
||||
public QFieldMetaData getFieldThrowing(String name) throws QException
|
||||
{
|
||||
Optional<QFieldMetaData> field = fieldList.stream().filter(f -> name.equals(f.getName())).findFirst();
|
||||
if(field.isEmpty())
|
||||
{
|
||||
throw (new QException("Could not find field [" + name + "] in function input meta data"));
|
||||
}
|
||||
return (field.get());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for fieldList
|
||||
**
|
||||
|
@ -0,0 +1,69 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.model.metadata.serialization;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import com.fasterxml.jackson.core.JacksonException;
|
||||
import com.fasterxml.jackson.core.JsonParser;
|
||||
import com.fasterxml.jackson.core.TreeNode;
|
||||
import com.fasterxml.jackson.databind.DeserializationContext;
|
||||
import com.fasterxml.jackson.databind.JsonDeserializer;
|
||||
import com.fasterxml.jackson.databind.node.NullNode;
|
||||
import com.fasterxml.jackson.databind.node.TextNode;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
|
||||
import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Jackson custom deserialization class, to return an appropriate sub-type of
|
||||
** A QBackendMetaData, based on the backendType specified within.
|
||||
*******************************************************************************/
|
||||
public class QFieldMappingDeserializer extends JsonDeserializer<AbstractQFieldMapping>
|
||||
{
|
||||
@Override
|
||||
public AbstractQFieldMapping deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JacksonException
|
||||
{
|
||||
TreeNode treeNode = jsonParser.readValueAsTree();
|
||||
|
||||
TreeNode sourceTypeTreeNode = treeNode.get("sourceType");
|
||||
if(sourceTypeTreeNode == null || sourceTypeTreeNode instanceof NullNode)
|
||||
{
|
||||
throw new IOException("Missing sourceType in serializedMapping");
|
||||
}
|
||||
|
||||
if(!(sourceTypeTreeNode instanceof TextNode textNode))
|
||||
{
|
||||
throw new IOException("sourceType is not a string value (is: " + sourceTypeTreeNode.getClass().getSimpleName() + ")");
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get the value of the backendType json node, and use it to look up the qBackendModule object //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
String backendType = textNode.asText();
|
||||
|
||||
QBackendModuleInterface backendModule = DeserializerUtils.getBackendModule(treeNode);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
@ -42,7 +42,7 @@ public class QBackendModuleDispatcher
|
||||
{
|
||||
private static final Logger LOG = LogManager.getLogger(QBackendModuleDispatcher.class);
|
||||
|
||||
private Map<String, String> backendTypeToModuleClassNameMap;
|
||||
private static Map<String, String> backendTypeToModuleClassNameMap;
|
||||
|
||||
|
||||
|
||||
@ -51,6 +51,21 @@ public class QBackendModuleDispatcher
|
||||
*******************************************************************************/
|
||||
public QBackendModuleDispatcher()
|
||||
{
|
||||
initBackendTypeToModuleClassNameMap();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static void initBackendTypeToModuleClassNameMap()
|
||||
{
|
||||
if(backendTypeToModuleClassNameMap != null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
backendTypeToModuleClassNameMap = new HashMap<>();
|
||||
|
||||
String[] moduleClassNames = new String[]
|
||||
@ -80,6 +95,22 @@ public class QBackendModuleDispatcher
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static void registerBackendModule(QBackendModuleInterface moduleInstance)
|
||||
{
|
||||
initBackendTypeToModuleClassNameMap();
|
||||
String backendType = moduleInstance.getBackendType();
|
||||
if(backendTypeToModuleClassNameMap.containsKey(backendType))
|
||||
{
|
||||
LOG.info("Overwriting backend type [" + backendType + "] with [" + moduleInstance.getClass() + "]");
|
||||
}
|
||||
backendTypeToModuleClassNameMap.put(backendType, moduleInstance.getClass().getName());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
@ -71,7 +71,6 @@ public class MockQueryAction implements QueryInterface
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
e.printStackTrace();
|
||||
throw new QException("Error executing query", e);
|
||||
}
|
||||
}
|
||||
|
@ -39,8 +39,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
public class BasicETLProcess
|
||||
{
|
||||
public static final String PROCESS_NAME = "etl.basic";
|
||||
public static final String FUNCTION_NAME_EXTRACT = "extract";
|
||||
public static final String FUNCTION_NAME_TRANSFORM = "transform";
|
||||
public static final String FUNCTION_NAME_LOAD = "load";
|
||||
public static final String FIELD_SOURCE_TABLE = "sourceTable";
|
||||
public static final String FIELD_DESTINATION_TABLE = "destinationTable";
|
||||
public static final String FIELD_MAPPING_JSON = "mappingJSON";
|
||||
public static final String FIELD_RECORD_COUNT = "recordCount";
|
||||
|
||||
|
||||
@ -51,7 +55,7 @@ public class BasicETLProcess
|
||||
public QProcessMetaData defineProcessMetaData()
|
||||
{
|
||||
QFunctionMetaData extractFunction = new QFunctionMetaData()
|
||||
.withName("extract")
|
||||
.withName(FUNCTION_NAME_EXTRACT)
|
||||
.withCode(new QCodeReference()
|
||||
.withName(BasicETLExtractFunction.class.getName())
|
||||
.withCodeType(QCodeType.JAVA)
|
||||
@ -59,8 +63,18 @@ public class BasicETLProcess
|
||||
.withInputData(new QFunctionInputMetaData()
|
||||
.addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)));
|
||||
|
||||
QFunctionMetaData transformFunction = new QFunctionMetaData()
|
||||
.withName(FUNCTION_NAME_TRANSFORM)
|
||||
.withCode(new QCodeReference()
|
||||
.withName(BasicETLTransformFunction.class.getName())
|
||||
.withCodeType(QCodeType.JAVA)
|
||||
.withCodeUsage(QCodeUsage.FUNCTION))
|
||||
.withInputData(new QFunctionInputMetaData()
|
||||
.addField(new QFieldMetaData(FIELD_MAPPING_JSON, QFieldType.STRING))
|
||||
.addField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)));
|
||||
|
||||
QFunctionMetaData loadFunction = new QFunctionMetaData()
|
||||
.withName("load")
|
||||
.withName(FUNCTION_NAME_LOAD)
|
||||
.withCode(new QCodeReference()
|
||||
.withName(BasicETLLoadFunction.class.getName())
|
||||
.withCodeType(QCodeType.JAVA)
|
||||
@ -73,6 +87,7 @@ public class BasicETLProcess
|
||||
return new QProcessMetaData()
|
||||
.withName(PROCESS_NAME)
|
||||
.addFunction(extractFunction)
|
||||
.addFunction(transformFunction)
|
||||
.addFunction(loadFunction);
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,99 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.adapters.JsonToQFieldMappingAdapter;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Function body for performing the Extract step of a basic ETL process.
|
||||
*******************************************************************************/
|
||||
public class BasicETLTransformFunction implements FunctionBody
|
||||
{
|
||||
@Override
|
||||
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// exit early with no-op if no records made it here, or if we don't have a mapping to use //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(CollectionUtils.nullSafeIsEmpty(runFunctionRequest.getRecords()))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
String mappingJSON = runFunctionRequest.getValueString(BasicETLProcess.FIELD_MAPPING_JSON);
|
||||
if(!StringUtils.hasContent(mappingJSON))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// require that the mapping be a key-based mapping (can't use indexes into qRecord values) //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
AbstractQFieldMapping<?> mapping = new JsonToQFieldMappingAdapter().buildMappingFromJson(mappingJSON);
|
||||
if(!(mapping instanceof QKeyBasedFieldMapping keyBasedFieldMapping))
|
||||
{
|
||||
throw (new QException("Mapping was not a Key-based mapping type. Was a : " + mapping.getClass().getName()));
|
||||
}
|
||||
|
||||
String tableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
|
||||
QTableMetaData table = runFunctionRequest.getInstance().getTable(tableName);
|
||||
List<QRecord> mappedRecords = applyMapping(runFunctionRequest.getRecords(), table, keyBasedFieldMapping);
|
||||
runFunctionResult.setRecords(mappedRecords);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private List<QRecord> applyMapping(List<QRecord> input, QTableMetaData table, QKeyBasedFieldMapping mapping)
|
||||
{
|
||||
List<QRecord> output = new ArrayList<>();
|
||||
for(QRecord inputRecord : input)
|
||||
{
|
||||
QRecord outputRecord = new QRecord();
|
||||
output.add(outputRecord);
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
{
|
||||
String fieldSource = mapping.getFieldSource(field.getName());
|
||||
outputRecord.setValue(field.getName(), inputRecord.getValue(fieldSource));
|
||||
}
|
||||
}
|
||||
return (output);
|
||||
}
|
||||
|
||||
}
|
@ -29,6 +29,8 @@ import java.io.Serializable;
|
||||
import java.util.Optional;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -36,6 +38,8 @@ import org.apache.commons.io.FileUtils;
|
||||
*******************************************************************************/
|
||||
public class TempFileStateProvider implements StateProviderInterface
|
||||
{
|
||||
private static final Logger LOG = LogManager.getLogger(TempFileStateProvider.class);
|
||||
|
||||
private static TempFileStateProvider instance;
|
||||
|
||||
|
||||
@ -76,8 +80,8 @@ public class TempFileStateProvider implements StateProviderInterface
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
// todo better
|
||||
e.printStackTrace();
|
||||
LOG.error("Error putting state into file", e);
|
||||
throw (new RuntimeException("Error storing state", e));
|
||||
}
|
||||
}
|
||||
|
||||
@ -98,9 +102,10 @@ public class TempFileStateProvider implements StateProviderInterface
|
||||
{
|
||||
return (Optional.empty());
|
||||
}
|
||||
catch(IOException ie)
|
||||
catch(IOException e)
|
||||
{
|
||||
throw new RuntimeException("Error loading state from file", ie);
|
||||
LOG.error("Error getting state from file", e);
|
||||
throw (new RuntimeException("Error retreiving state", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
29
src/main/resources/log4j2.xml
Normal file
29
src/main/resources/log4j2.xml
Normal file
@ -0,0 +1,29 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Configuration>
|
||||
<Properties>
|
||||
<Property name="LOG_PATTERN">%highlight{%d{ISO8601} | %-6r | %-5p | %markerSimpleName | %c{1} | %t{1} | %m%n}{FATAL=red, ERROR=red, WARN=blue, INFO=green, METRICS=magenta, DEBUG=cyan, TRACE=white}</Property>
|
||||
</Properties>
|
||||
<CustomLevels>
|
||||
<CustomLevel name="METRICS" intLevel="450"/>
|
||||
</CustomLevels>
|
||||
<Appenders>
|
||||
<File name="FullLogFile" fileName="log/full_log.log">
|
||||
<LevelRangeFilter minLevel="ERROR" maxLevel="all" onMatch="ACCEPT" onMismatch="DENY"/>
|
||||
<PatternLayout pattern="${LOG_PATTERN}"/>
|
||||
</File>
|
||||
<Console name="STDOUT" target="SYSTEM_OUT">
|
||||
<LevelRangeFilter minLevel="ERROR" maxLevel="METRICS" onMatch="ACCEPT" onMismatch="DENY"/>
|
||||
<PatternLayout pattern="${LOG_PATTERN}"/>
|
||||
</Console>
|
||||
</Appenders>
|
||||
<Loggers>
|
||||
<Logger name="org.apache.log4j.xml" additivity="false">
|
||||
</Logger>
|
||||
<Root level="all">
|
||||
<AppenderRef ref="STDOUT"/>
|
||||
<!-- Uncomment only if you need the full_log.
|
||||
<AppenderRef ref="FullLogFile" />
|
||||
-->
|
||||
</Root>
|
||||
</Loggers>
|
||||
</Configuration>
|
@ -27,6 +27,7 @@ import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFi
|
||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
@ -230,4 +231,54 @@ class CsvToQRecordAdapterTest
|
||||
assertEquals("1981-01-01", qRecord2.getValue("birthDate"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** In this test - we've got CSV data with duplicated header names.
|
||||
** In our mapping, we're seeing the suffixes of " 2" and " 3" addd to those
|
||||
** header names on the RHS.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test_duplicatedColumnHeaders()
|
||||
{
|
||||
QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping()
|
||||
.withMapping("id", "id")
|
||||
.withMapping("createDate", "date")
|
||||
.withMapping("modifyDate", "date 2")
|
||||
.withMapping("firstName", "name")
|
||||
.withMapping("lastName", "name 2")
|
||||
.withMapping("birthDate", "date 3")
|
||||
.withMapping("email", "email");
|
||||
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv("""
|
||||
id,date,date,name,name,date,email
|
||||
1,2022-06-26,2022-06-26,John,Doe,1980-01-01,john@kingsrook.com
|
||||
""", TestUtils.defineTablePerson(), mapping);
|
||||
assertNotNull(qRecords);
|
||||
assertEquals(1, qRecords.size());
|
||||
QRecord qRecord1 = qRecords.get(0);
|
||||
assertEquals("John", qRecord1.getValue("firstName"));
|
||||
assertEquals("Doe", qRecord1.getValue("lastName"));
|
||||
assertEquals("1980-01-01", qRecord1.getValue("birthDate"));
|
||||
assertEquals("2022-06-26", qRecord1.getValue("createDate"));
|
||||
assertEquals("2022-06-26", qRecord1.getValue("modifyDate"));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testMakeHeadersUnique()
|
||||
{
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
Assertions.assertEquals(List.of("A", "B", "C"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C")));
|
||||
Assertions.assertEquals(List.of("A", "B", "C", "C 2", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C", "C", "C")));
|
||||
Assertions.assertEquals(List.of("C", "A", "C 2", "B", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("C", "A", "C", "B", "C")));
|
||||
Assertions.assertEquals(List.of("A", "B", "C", "C 2", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C", "C 2", "C")));
|
||||
Assertions.assertEquals(List.of("A", "B", "C", "C 2", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C", "C 2", "C 3")));
|
||||
// todo - this is what the method header comment means when it says we don't handle all cases well...
|
||||
// Assertions.assertEquals(List.of("A", "B", "C", "C 2", "C 3"), csvToQRecordAdapter.makeHeadersUnique(List.of("A", "B", "C 2", "C", "C 3")));
|
||||
}
|
||||
}
|
||||
|
@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.adapters;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFieldMapping;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
import org.assertj.core.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
@ -130,6 +132,38 @@ class JsonToQFieldMappingAdapterTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test_deserializeSerializedQKeyBasedFieldMapping()
|
||||
{
|
||||
QIndexBasedFieldMapping original = new QIndexBasedFieldMapping()
|
||||
.withMapping("foo", 0)
|
||||
.withMapping("bar", 1);
|
||||
String json = JsonUtils.toJson(original);
|
||||
AbstractQFieldMapping<?> deserialized = new JsonToQFieldMappingAdapter().buildMappingFromJson(json);
|
||||
Assertions.assertThat(deserialized).usingRecursiveComparison().isEqualTo(original);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test_deserializeSerializedQIndexBasedFieldMapping()
|
||||
{
|
||||
QKeyBasedFieldMapping original = new QKeyBasedFieldMapping()
|
||||
.withMapping("foo", "Fu")
|
||||
.withMapping("bar", "Bahr");
|
||||
String json = JsonUtils.toJson(original);
|
||||
AbstractQFieldMapping<?> deserialized = new JsonToQFieldMappingAdapter().buildMappingFromJson(json);
|
||||
Assertions.assertThat(deserialized).usingRecursiveComparison().isEqualTo(original);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -140,6 +174,7 @@ class JsonToQFieldMappingAdapterTest
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -159,7 +194,7 @@ class JsonToQFieldMappingAdapterTest
|
||||
try
|
||||
{
|
||||
JsonToQFieldMappingAdapter jsonToQFieldMappingAdapter = new JsonToQFieldMappingAdapter();
|
||||
AbstractQFieldMapping<?> mapping = jsonToQFieldMappingAdapter.buildMappingFromJson(json);
|
||||
AbstractQFieldMapping<?> mapping = jsonToQFieldMappingAdapter.buildMappingFromJson(json);
|
||||
System.out.println(mapping);
|
||||
}
|
||||
catch(IllegalArgumentException iae)
|
||||
|
@ -0,0 +1,55 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.model.metadata;
|
||||
|
||||
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for QSecretReader
|
||||
*******************************************************************************/
|
||||
class QSecretReaderTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testReadSecret()
|
||||
{
|
||||
QSecretReader secretReader = new QSecretReader();
|
||||
String key = "CUSTOM_PROPERTY";
|
||||
String value = "ABCD-9876";
|
||||
secretReader.setCustomEnvironment(Map.of(key, value));
|
||||
|
||||
assertNull(secretReader.readSecret(null));
|
||||
assertEquals("foo", secretReader.readSecret("foo"));
|
||||
assertNull(secretReader.readSecret("${env.NOT-" + key + "}"));
|
||||
assertEquals(value, secretReader.readSecret("${env." + key + "}"));
|
||||
assertEquals("${env.NOT-" + key, secretReader.readSecret("${env.NOT-" + key));
|
||||
}
|
||||
|
||||
}
|
@ -26,6 +26,8 @@ import com.kingsrook.qqq.backend.core.actions.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessResult;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
@ -40,7 +42,7 @@ class BasicETLProcessTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
** Simplest happy path
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test() throws QException
|
||||
@ -50,6 +52,7 @@ class BasicETLProcessTest
|
||||
request.setProcessName(BasicETLProcess.PROCESS_NAME);
|
||||
request.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.defineTablePerson().getName());
|
||||
request.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.definePersonFileTable().getName());
|
||||
request.addValue(BasicETLProcess.FIELD_MAPPING_JSON, "");
|
||||
|
||||
RunProcessResult result = new RunProcessAction().execute(request);
|
||||
assertNotNull(result);
|
||||
@ -58,4 +61,30 @@ class BasicETLProcessTest
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Basic example of doing a mapping transformation
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testMappingTransformation() throws QException
|
||||
{
|
||||
RunProcessRequest request = new RunProcessRequest(TestUtils.defineInstance());
|
||||
request.setSession(TestUtils.getMockSession());
|
||||
request.setProcessName(BasicETLProcess.PROCESS_NAME);
|
||||
request.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.definePersonFileTable().getName());
|
||||
request.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.defineTableIdAndNameOnly().getName());
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
// define our mapping from destination-table field names to source-table field names //
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping().withMapping("name", "firstName");
|
||||
// request.addValue(BasicETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping.getMapping()));
|
||||
request.addValue(BasicETLProcess.FIELD_MAPPING_JSON, JsonUtils.toJson(mapping));
|
||||
|
||||
RunProcessResult result = new RunProcessAction().execute(request);
|
||||
assertNotNull(result);
|
||||
assertNull(result.getError());
|
||||
assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process");
|
||||
}
|
||||
|
||||
}
|
@ -68,6 +68,7 @@ public class TestUtils
|
||||
qInstance.addBackend(defineBackend());
|
||||
qInstance.addTable(defineTablePerson());
|
||||
qInstance.addTable(definePersonFileTable());
|
||||
qInstance.addTable(defineTableIdAndNameOnly());
|
||||
qInstance.addPossibleValueSource(defineStatesPossibleValueSource());
|
||||
qInstance.addProcess(defineProcessGreetPeople());
|
||||
qInstance.addProcess(defineProcessAddToPeoplesAge());
|
||||
@ -153,6 +154,22 @@ public class TestUtils
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Define simple table with just an id and name
|
||||
*******************************************************************************/
|
||||
public static QTableMetaData defineTableIdAndNameOnly()
|
||||
{
|
||||
return new QTableMetaData()
|
||||
.withName("idAndNameOnly")
|
||||
.withLabel("Id and Name Only")
|
||||
.withBackendName(DEFAULT_BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Define the 'greet people' process
|
||||
*******************************************************************************/
|
||||
|
Reference in New Issue
Block a user