mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 13:10:44 +00:00
QQQ-14 checkpoint, pre-demo
This commit is contained in:
@ -19,37 +19,22 @@
|
|||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package com.kingsrook.qqq.backend.core.model.metadata;
|
package com.kingsrook.qqq.backend.core.instances;
|
||||||
|
|
||||||
|
|
||||||
import java.util.Map;
|
import java.lang.annotation.ElementType;
|
||||||
import org.junit.jupiter.api.Test;
|
import java.lang.annotation.Retention;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import java.lang.annotation.RetentionPolicy;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
import java.lang.annotation.Target;
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
** Unit test for QSecretReader
|
** Class-level annotation to declare what fields should run through the variable
|
||||||
|
** interpreter - e.g., to be replaced with env-var values at run-time.
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
class QSecretReaderTest
|
@Retention(RetentionPolicy.RUNTIME)
|
||||||
|
@Target(ElementType.TYPE)
|
||||||
|
public @interface InterpretableFields
|
||||||
{
|
{
|
||||||
|
String[] fieldNames() default {};
|
||||||
/*******************************************************************************
|
|
||||||
**
|
|
||||||
*******************************************************************************/
|
|
||||||
@Test
|
|
||||||
void testReadSecret()
|
|
||||||
{
|
|
||||||
QSecretReader secretReader = new QSecretReader();
|
|
||||||
String key = "CUSTOM_PROPERTY";
|
|
||||||
String value = "ABCD-9876";
|
|
||||||
secretReader.setCustomEnvironment(Map.of(key, value));
|
|
||||||
|
|
||||||
assertNull(secretReader.readSecret(null));
|
|
||||||
assertEquals("foo", secretReader.readSecret("foo"));
|
|
||||||
assertNull(secretReader.readSecret("${env.NOT-" + key + "}"));
|
|
||||||
assertEquals(value, secretReader.readSecret("${env." + key + "}"));
|
|
||||||
assertEquals("${env.NOT-" + key, secretReader.readSecret("${env.NOT-" + key));
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
@ -0,0 +1,172 @@
|
|||||||
|
/*
|
||||||
|
* QQQ - Low-code Application Framework for Engineers.
|
||||||
|
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||||
|
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||||
|
* contact@kingsrook.com
|
||||||
|
* https://github.com/Kingsrook/
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.kingsrook.qqq.backend.core.instances;
|
||||||
|
|
||||||
|
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** To avoid having secrets (passwords, access keys, etc) committed into meta data
|
||||||
|
** files, as well as to just let some meta data not be hard-coded, this class is
|
||||||
|
** used by the Enricher to "promote" values, such as ${env.ACCESS_KEY}
|
||||||
|
** to be read from the environment (or other secret providers (to be implemented)).
|
||||||
|
**
|
||||||
|
** Supported syntax / value sources are:
|
||||||
|
** ${env.VAR} = system environment variables, e.g., export VAR=val
|
||||||
|
** ${prop.VAR} = properties, e.g., -DVAR=val
|
||||||
|
** ${literal.VAR} = get back a literal "VAR" (in case VAR matches some of the other supported syntax in here)
|
||||||
|
*******************************************************************************/
|
||||||
|
public class QMetaDataVariableInterpreter
|
||||||
|
{
|
||||||
|
private static final Logger LOG = LogManager.getLogger(QMetaDataVariableInterpreter.class);
|
||||||
|
|
||||||
|
private Map<String, String> customEnvironment;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public void interpretObject(Object o) throws QException
|
||||||
|
{
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// get the InterpretableFields from the object's class - exiting if the annotation isn't present //
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
InterpretableFields interpretableFields = o.getClass().getAnnotation(InterpretableFields.class);
|
||||||
|
if(interpretableFields == null)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////
|
||||||
|
// iterate over interpretable fields, interpreting each //
|
||||||
|
//////////////////////////////////////////////////////////
|
||||||
|
for(String fieldName : interpretableFields.fieldNames())
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// get the getter & setter methods for the field (getMethod will throw if not found) //
|
||||||
|
// enforce Strings-only at this time. //
|
||||||
|
///////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
String fieldNameUcFirst = fieldName.substring(0, 1).toUpperCase(Locale.ROOT) + fieldName.substring(1);
|
||||||
|
Method getter = o.getClass().getMethod("get" + fieldNameUcFirst);
|
||||||
|
Class<?> fieldType = getter.getReturnType();
|
||||||
|
if(!fieldType.equals(String.class))
|
||||||
|
{
|
||||||
|
throw new QException("Interpretable field: " + fieldName + " on class " + o.getClass() + " is not a String (which is required at this time)");
|
||||||
|
}
|
||||||
|
Method setter = o.getClass().getMethod("set" + fieldNameUcFirst, fieldType);
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// get the value - if it's null, move on, else, interpret it, and put it back in the object //
|
||||||
|
//////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
|
Object value = getter.invoke(o);
|
||||||
|
if(value == null)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String interpreted = interpret((String) value);
|
||||||
|
setter.invoke(o, interpreted);
|
||||||
|
}
|
||||||
|
catch(Exception e)
|
||||||
|
{
|
||||||
|
throw (new QException("Error interpreting variables in object " + o, e));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Interpret a value string, which may be a variable, into its run-time value.
|
||||||
|
**
|
||||||
|
** If input is null, output is null.
|
||||||
|
** If input looks like ${env.X}, then the return value is the value of the env variable 'X'
|
||||||
|
** If input looks like ${prop.X}, then the return value is the value of the system property 'X'
|
||||||
|
** If input looks like ${literal.X}, then the return value is the literal 'X'
|
||||||
|
** - used if you really want to get back the literal value, ${env.X}, for example.
|
||||||
|
** Else the output is the input.
|
||||||
|
*******************************************************************************/
|
||||||
|
public String interpret(String value)
|
||||||
|
{
|
||||||
|
if(value == null)
|
||||||
|
{
|
||||||
|
return (null);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(value.startsWith("${env.") && value.endsWith("}"))
|
||||||
|
{
|
||||||
|
String envVarName = value.substring(6).replaceFirst("}$", "");
|
||||||
|
String envValue = getEnvironment().get(envVarName);
|
||||||
|
return (envValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(value.startsWith("${prop.") && value.endsWith("}"))
|
||||||
|
{
|
||||||
|
String propertyName = value.substring(7).replaceFirst("}$", "");
|
||||||
|
String propertyValue = System.getProperty(propertyName);
|
||||||
|
return (propertyValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(value.startsWith("${literal.") && value.endsWith("}"))
|
||||||
|
{
|
||||||
|
String literalValue = value.substring(10).replaceFirst("}$", "");
|
||||||
|
return (literalValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (value);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Setter for customEnvironment - protected - meant to be called (at least at this
|
||||||
|
** time), only in unit test
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
protected void setCustomEnvironment(Map<String, String> customEnvironment)
|
||||||
|
{
|
||||||
|
this.customEnvironment = customEnvironment;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
private Map<String, String> getEnvironment()
|
||||||
|
{
|
||||||
|
if(this.customEnvironment != null)
|
||||||
|
{
|
||||||
|
return (this.customEnvironment);
|
||||||
|
}
|
||||||
|
|
||||||
|
return System.getenv();
|
||||||
|
}
|
||||||
|
}
|
@ -36,7 +36,21 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
|||||||
public class RunProcessResult extends AbstractQResult
|
public class RunProcessResult extends AbstractQResult
|
||||||
{
|
{
|
||||||
private ProcessState processState;
|
private ProcessState processState;
|
||||||
private String error;
|
private String error;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Override
|
||||||
|
public String toString()
|
||||||
|
{
|
||||||
|
return "RunProcessResult{error='" + error
|
||||||
|
+ ",records.size()=" + (processState == null ? null : processState.getRecords().size())
|
||||||
|
+ ",values=" + (processState == null ? null : processState.getValues())
|
||||||
|
+ "}";
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@ -1,89 +0,0 @@
|
|||||||
/*
|
|
||||||
* QQQ - Low-code Application Framework for Engineers.
|
|
||||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
|
||||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
|
||||||
* contact@kingsrook.com
|
|
||||||
* https://github.com/Kingsrook/
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU Affero General Public License as
|
|
||||||
* published by the Free Software Foundation, either version 3 of the
|
|
||||||
* License, or (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
* GNU Affero General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package com.kingsrook.qqq.backend.core.model.metadata;
|
|
||||||
|
|
||||||
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
|
||||||
** To avoid having secrets (passwords, access keys, etc) committed into meta data
|
|
||||||
** files, this class is used by the Enricher to "promote" values, such as ${env.ACCESS_KEY}
|
|
||||||
** to be read from the environment (or other secret providers (to be implemented)).
|
|
||||||
*******************************************************************************/
|
|
||||||
public class QSecretReader
|
|
||||||
{
|
|
||||||
private Map<String, String> customEnvironment;
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
|
||||||
** Translate a secret.
|
|
||||||
**
|
|
||||||
** If input is null, output is null.
|
|
||||||
** If input looks like ${env.X}, then the return value is the value of the env variable 'X'
|
|
||||||
** Else the output is the input.
|
|
||||||
*******************************************************************************/
|
|
||||||
public String readSecret(String value)
|
|
||||||
{
|
|
||||||
if(value == null)
|
|
||||||
{
|
|
||||||
return (null);
|
|
||||||
}
|
|
||||||
|
|
||||||
if(value.startsWith("${env.") && value.endsWith("}"))
|
|
||||||
{
|
|
||||||
String envVarName = value.substring(6).replaceFirst("}$", "");
|
|
||||||
String envValue = getEnvironment().get(envVarName);
|
|
||||||
return (envValue);
|
|
||||||
}
|
|
||||||
|
|
||||||
return (value);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
|
||||||
** Setter for customEnvironment - protected - meant to be called (at least at this
|
|
||||||
** time), only in unit test
|
|
||||||
**
|
|
||||||
*******************************************************************************/
|
|
||||||
protected void setCustomEnvironment(Map<String, String> customEnvironment)
|
|
||||||
{
|
|
||||||
this.customEnvironment = customEnvironment;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
|
||||||
**
|
|
||||||
*******************************************************************************/
|
|
||||||
private Map<String, String> getEnvironment()
|
|
||||||
{
|
|
||||||
if(this.customEnvironment != null)
|
|
||||||
{
|
|
||||||
return (this.customEnvironment);
|
|
||||||
}
|
|
||||||
|
|
||||||
return System.getenv();
|
|
||||||
}
|
|
||||||
}
|
|
@ -22,6 +22,7 @@
|
|||||||
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||||
|
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import com.kingsrook.qqq.backend.core.actions.InsertAction;
|
import com.kingsrook.qqq.backend.core.actions.InsertAction;
|
||||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
@ -32,6 +33,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest
|
|||||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
|
||||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
@ -39,13 +42,22 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
|||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public class BasicETLLoadFunction implements FunctionBody
|
public class BasicETLLoadFunction implements FunctionBody
|
||||||
{
|
{
|
||||||
|
private static final Logger LOG = LogManager.getLogger(BasicETLLoadFunction.class);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
@Override
|
@Override
|
||||||
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
|
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
|
||||||
{
|
{
|
||||||
//////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////
|
||||||
// exit early with no-op if no records made it here //
|
// exit early with no-op if no records made it here //
|
||||||
//////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////
|
||||||
if(CollectionUtils.nullSafeIsEmpty(runFunctionRequest.getRecords()))
|
List<QRecord> inputRecords = runFunctionRequest.getRecords();
|
||||||
|
LOG.info("Received [" + inputRecords.size() + "] records to load");
|
||||||
|
if(CollectionUtils.nullSafeIsEmpty(inputRecords))
|
||||||
{
|
{
|
||||||
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, 0);
|
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, 0);
|
||||||
return;
|
return;
|
||||||
@ -55,7 +67,7 @@ public class BasicETLLoadFunction implements FunctionBody
|
|||||||
// put the destination table name in all records being inserted //
|
// put the destination table name in all records being inserted //
|
||||||
//////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////
|
||||||
String table = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
|
String table = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
|
||||||
for(QRecord record : runFunctionRequest.getRecords())
|
for(QRecord record : inputRecords)
|
||||||
{
|
{
|
||||||
record.setTableName(table);
|
record.setTableName(table);
|
||||||
}
|
}
|
||||||
@ -63,16 +75,26 @@ public class BasicETLLoadFunction implements FunctionBody
|
|||||||
//////////////////////////////////////////
|
//////////////////////////////////////////
|
||||||
// run an insert request on the records //
|
// run an insert request on the records //
|
||||||
//////////////////////////////////////////
|
//////////////////////////////////////////
|
||||||
InsertRequest insertRequest = new InsertRequest(runFunctionRequest.getInstance());
|
int recordsInserted = 0;
|
||||||
insertRequest.setSession(runFunctionRequest.getSession());
|
List<QRecord> outputRecords = new ArrayList<>();
|
||||||
insertRequest.setTableName(table);
|
int pageSize = 1000; // todo - make this a field?
|
||||||
insertRequest.setRecords(runFunctionRequest.getRecords());
|
|
||||||
|
|
||||||
InsertAction insertAction = new InsertAction();
|
for(List<QRecord> page : CollectionUtils.getPages(inputRecords, pageSize))
|
||||||
InsertResult insertResult = insertAction.execute(insertRequest);
|
{
|
||||||
|
LOG.info("Inserting a page of [" + page.size() + "] records. Progress: " + recordsInserted + " loaded out of " + inputRecords.size() + " total");
|
||||||
|
InsertRequest insertRequest = new InsertRequest(runFunctionRequest.getInstance());
|
||||||
|
insertRequest.setSession(runFunctionRequest.getSession());
|
||||||
|
insertRequest.setTableName(table);
|
||||||
|
insertRequest.setRecords(page);
|
||||||
|
|
||||||
runFunctionResult.setRecords(insertResult.getRecords());
|
InsertAction insertAction = new InsertAction();
|
||||||
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, insertResult.getRecords().size());
|
InsertResult insertResult = insertAction.execute(insertRequest);
|
||||||
|
outputRecords.addAll(insertResult.getRecords());
|
||||||
|
|
||||||
|
recordsInserted += insertResult.getRecords().size();
|
||||||
|
}
|
||||||
|
runFunctionResult.setRecords(outputRecords);
|
||||||
|
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, recordsInserted);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -22,6 +22,7 @@
|
|||||||
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||||
|
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import com.kingsrook.qqq.backend.core.adapters.JsonToQFieldMappingAdapter;
|
import com.kingsrook.qqq.backend.core.adapters.JsonToQFieldMappingAdapter;
|
||||||
@ -33,9 +34,12 @@ import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFiel
|
|||||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
|
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
|
||||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||||
|
import org.apache.logging.log4j.LogManager;
|
||||||
|
import org.apache.logging.log4j.Logger;
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
@ -43,6 +47,8 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
|||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public class BasicETLTransformFunction implements FunctionBody
|
public class BasicETLTransformFunction implements FunctionBody
|
||||||
{
|
{
|
||||||
|
private static final Logger LOG = LogManager.getLogger(BasicETLTransformFunction.class);
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
|
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
|
||||||
{
|
{
|
||||||
@ -72,11 +78,49 @@ public class BasicETLTransformFunction implements FunctionBody
|
|||||||
String tableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
|
String tableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
|
||||||
QTableMetaData table = runFunctionRequest.getInstance().getTable(tableName);
|
QTableMetaData table = runFunctionRequest.getInstance().getTable(tableName);
|
||||||
List<QRecord> mappedRecords = applyMapping(runFunctionRequest.getRecords(), table, keyBasedFieldMapping);
|
List<QRecord> mappedRecords = applyMapping(runFunctionRequest.getRecords(), table, keyBasedFieldMapping);
|
||||||
|
|
||||||
|
removeNonNumericValuesFromMappedRecords(table, mappedRecords);
|
||||||
|
|
||||||
runFunctionResult.setRecords(mappedRecords);
|
runFunctionResult.setRecords(mappedRecords);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
private void removeNonNumericValuesFromMappedRecords(QTableMetaData table, List<QRecord> records)
|
||||||
|
{
|
||||||
|
for(QRecord record : records)
|
||||||
|
{
|
||||||
|
for(QFieldMetaData field : table.getFields().values())
|
||||||
|
{
|
||||||
|
Object value = record.getValue(field.getName());
|
||||||
|
if(value != null && StringUtils.hasContent(String.valueOf(value)))
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if(field.getType().equals(QFieldType.INTEGER))
|
||||||
|
{
|
||||||
|
Integer.parseInt(String.valueOf(value));
|
||||||
|
}
|
||||||
|
else if(field.getType().equals(QFieldType.DECIMAL))
|
||||||
|
{
|
||||||
|
new BigDecimal(String.valueOf(value));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch(NumberFormatException nfe)
|
||||||
|
{
|
||||||
|
LOG.info("Removing non-numeric value [" + value + "] from field [" + field.getName() + "]");
|
||||||
|
record.setValue(field.getName(), null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
**
|
**
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
|
@ -0,0 +1,281 @@
|
|||||||
|
/*
|
||||||
|
* QQQ - Low-code Application Framework for Engineers.
|
||||||
|
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||||
|
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||||
|
* contact@kingsrook.com
|
||||||
|
* https://github.com/Kingsrook/
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.kingsrook.qqq.backend.core.instances;
|
||||||
|
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
|
import org.junit.jupiter.api.AfterEach;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Unit test for QSecretReader
|
||||||
|
*******************************************************************************/
|
||||||
|
class QMetaDataVariableInterpreterTest
|
||||||
|
{
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@BeforeEach
|
||||||
|
public void beforeEach()
|
||||||
|
{
|
||||||
|
System.setProperty("username", "joe");
|
||||||
|
System.setProperty("password", "b1d3n");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@AfterEach
|
||||||
|
public void afterEach()
|
||||||
|
{
|
||||||
|
System.clearProperty("username");
|
||||||
|
System.clearProperty("password");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
void testInterpretObject() throws QException
|
||||||
|
{
|
||||||
|
GoodTestClass goodTestClass = new GoodTestClass();
|
||||||
|
goodTestClass.setUsername("${prop.username}");
|
||||||
|
goodTestClass.setPassword("${prop.password}");
|
||||||
|
|
||||||
|
new QMetaDataVariableInterpreter().interpretObject(goodTestClass);
|
||||||
|
|
||||||
|
assertEquals("joe", goodTestClass.getUsername());
|
||||||
|
assertEquals("b1d3n", goodTestClass.getPassword());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
void testBadAnnotatedObjects()
|
||||||
|
{
|
||||||
|
assertThrows(QException.class, () -> new QMetaDataVariableInterpreter().interpretObject(new BadTestClassAnnotatedInteger()));
|
||||||
|
assertThrows(QException.class, () -> new QMetaDataVariableInterpreter().interpretObject(new BadTestClassNoGetter()));
|
||||||
|
assertThrows(QException.class, () -> new QMetaDataVariableInterpreter().interpretObject(new BadTestClassNoSetter()));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
void testInterpretFromEnvironment()
|
||||||
|
{
|
||||||
|
QMetaDataVariableInterpreter secretReader = new QMetaDataVariableInterpreter();
|
||||||
|
String key = "CUSTOM_PROPERTY";
|
||||||
|
String value = "ABCD-9876";
|
||||||
|
secretReader.setCustomEnvironment(Map.of(key, value));
|
||||||
|
|
||||||
|
assertNull(secretReader.interpret(null));
|
||||||
|
assertEquals("foo", secretReader.interpret("foo"));
|
||||||
|
assertNull(secretReader.interpret("${env.NOT-" + key + "}"));
|
||||||
|
assertEquals(value, secretReader.interpret("${env." + key + "}"));
|
||||||
|
assertEquals("${env.NOT-" + key, secretReader.interpret("${env.NOT-" + key));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
void testInterpretFromProperties()
|
||||||
|
{
|
||||||
|
QMetaDataVariableInterpreter secretReader = new QMetaDataVariableInterpreter();
|
||||||
|
String key = "MY_PROPERTY";
|
||||||
|
String value = "WXYZ-6789";
|
||||||
|
System.setProperty(key, value);
|
||||||
|
|
||||||
|
assertNull(secretReader.interpret(null));
|
||||||
|
assertEquals("foo", secretReader.interpret("foo"));
|
||||||
|
assertNull(secretReader.interpret("${prop.NOT-" + key + "}"));
|
||||||
|
assertEquals(value, secretReader.interpret("${prop." + key + "}"));
|
||||||
|
assertEquals("${prop.NOT-" + key, secretReader.interpret("${prop.NOT-" + key));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
void testInterpretLiterals()
|
||||||
|
{
|
||||||
|
QMetaDataVariableInterpreter secretReader = new QMetaDataVariableInterpreter();
|
||||||
|
assertEquals("${env.X}", secretReader.interpret("${literal.${env.X}}"));
|
||||||
|
assertEquals("${prop.X}", secretReader.interpret("${literal.${prop.X}}"));
|
||||||
|
assertEquals("${literal.X}", secretReader.interpret("${literal.${literal.X}}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@InterpretableFields(fieldNames = { "username", "password" })
|
||||||
|
public static class GoodTestClass
|
||||||
|
{
|
||||||
|
private String username;
|
||||||
|
private String password;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Getter for username
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public String getUsername()
|
||||||
|
{
|
||||||
|
return username;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Setter for username
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public void setUsername(String username)
|
||||||
|
{
|
||||||
|
this.username = username;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Getter for password
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public String getPassword()
|
||||||
|
{
|
||||||
|
return password;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Setter for password
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public void setPassword(String password)
|
||||||
|
{
|
||||||
|
this.password = password;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@InterpretableFields(fieldNames = { "port" })
|
||||||
|
public static class BadTestClassAnnotatedInteger
|
||||||
|
{
|
||||||
|
private Integer port;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Getter for port
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public Integer getPort()
|
||||||
|
{
|
||||||
|
return port;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Setter for port
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public void setPort(Integer port)
|
||||||
|
{
|
||||||
|
this.port = port;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@InterpretableFields(fieldNames = { "foo" })
|
||||||
|
public static class BadTestClassNoGetter
|
||||||
|
{
|
||||||
|
private String foo;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Setter for foo
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public void setFoo(String foo)
|
||||||
|
{
|
||||||
|
this.foo = foo;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@InterpretableFields(fieldNames = { "foo" })
|
||||||
|
public static class BadTestClassNoSetter
|
||||||
|
{
|
||||||
|
private String foo;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Getter for foo
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public String getFoo()
|
||||||
|
{
|
||||||
|
return foo;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Reference in New Issue
Block a user