QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-14 checkpoint, pre-demo

Browse Source
This commit is contained in:
darin.kelkhoff
2022-06-29 10:12:43 -05:00
parent 28579cc52c
commit 656589b0df
7 changed files with 556 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/test/java/com/kingsrook/qqq/backend/core/model/metadata/QSecretReaderTest.java → src/main/java/com/kingsrook/qqq/backend/core/instances/InterpretableFields.java
View File

@ -19,37 +19,22 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata;
package com.kingsrook.qqq.backend.core.instances;
import java.util.Map;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/*******************************************************************************
** Unit test for QSecretReader
** Class-level annotation to declare what fields should run through the variable
** interpreter - e.g., to be replaced with env-var values at run-time.
*******************************************************************************/
class QSecretReaderTest
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
public @interface InterpretableFields
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testReadSecret()
{
QSecretReader secretReader = new QSecretReader();
String key = "CUSTOM_PROPERTY";
String value = "ABCD-9876";
secretReader.setCustomEnvironment(Map.of(key, value));
assertNull(secretReader.readSecret(null));
assertEquals("foo", secretReader.readSecret("foo"));
assertNull(secretReader.readSecret("${env.NOT-" + key + "}"));
assertEquals(value, secretReader.readSecret("${env." + key + "}"));
assertEquals("${env.NOT-" + key, secretReader.readSecret("${env.NOT-" + key));
}
}
String[] fieldNames() default {};
}

172
src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java Normal file
View File

@ -0,0 +1,172 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.instances;
import java.lang.reflect.Method;
import java.util.Locale;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** To avoid having secrets (passwords, access keys, etc) committed into meta data
** files, as well as to just let some meta data not be hard-coded, this class is
** used by the Enricher to "promote" values, such as ${env.ACCESS_KEY}
** to be read from the environment (or other secret providers (to be implemented)).
**
** Supported syntax / value sources are:
** ${env.VAR} = system environment variables, e.g., export VAR=val
** ${prop.VAR} = properties, e.g., -DVAR=val
** ${literal.VAR} = get back a literal "VAR" (in case VAR matches some of the other supported syntax in here)
*******************************************************************************/
public class QMetaDataVariableInterpreter
{
private static final Logger LOG = LogManager.getLogger(QMetaDataVariableInterpreter.class);
private Map<String, String> customEnvironment;
/*******************************************************************************
**
*******************************************************************************/
public void interpretObject(Object o) throws QException
{
///////////////////////////////////////////////////////////////////////////////////////////////////
// get the InterpretableFields from the object's class - exiting if the annotation isn't present //
///////////////////////////////////////////////////////////////////////////////////////////////////
InterpretableFields interpretableFields = o.getClass().getAnnotation(InterpretableFields.class);
if(interpretableFields == null)
{
return;
}
//////////////////////////////////////////////////////////
// iterate over interpretable fields, interpreting each //
//////////////////////////////////////////////////////////
for(String fieldName : interpretableFields.fieldNames())
{
try
{
///////////////////////////////////////////////////////////////////////////////////////
// get the getter & setter methods for the field (getMethod will throw if not found) //
// enforce Strings-only at this time. //
///////////////////////////////////////////////////////////////////////////////////////
String fieldNameUcFirst = fieldName.substring(0, 1).toUpperCase(Locale.ROOT) + fieldName.substring(1);
Method getter = o.getClass().getMethod("get" + fieldNameUcFirst);
Class<?> fieldType = getter.getReturnType();
if(!fieldType.equals(String.class))
{
throw new QException("Interpretable field: " + fieldName + " on class " + o.getClass() + " is not a String (which is required at this time)");
}
Method setter = o.getClass().getMethod("set" + fieldNameUcFirst, fieldType);
//////////////////////////////////////////////////////////////////////////////////////////////
// get the value - if it's null, move on, else, interpret it, and put it back in the object //
//////////////////////////////////////////////////////////////////////////////////////////////
Object value = getter.invoke(o);
if(value == null)
{
continue;
}
String interpreted = interpret((String) value);
setter.invoke(o, interpreted);
}
catch(Exception e)
{
throw (new QException("Error interpreting variables in object " + o, e));
}
}
}
/*******************************************************************************
** Interpret a value string, which may be a variable, into its run-time value.
**
** If input is null, output is null.
** If input looks like ${env.X}, then the return value is the value of the env variable 'X'
** If input looks like ${prop.X}, then the return value is the value of the system property 'X'
** If input looks like ${literal.X}, then the return value is the literal 'X'
** - used if you really want to get back the literal value, ${env.X}, for example.
** Else the output is the input.
*******************************************************************************/
public String interpret(String value)
{
if(value == null)
{
return (null);
}
if(value.startsWith("${env.") && value.endsWith("}"))
{
String envVarName = value.substring(6).replaceFirst("}$", "");
String envValue = getEnvironment().get(envVarName);
return (envValue);
}
if(value.startsWith("${prop.") && value.endsWith("}"))
{
String propertyName = value.substring(7).replaceFirst("}$", "");
String propertyValue = System.getProperty(propertyName);
return (propertyValue);
}
if(value.startsWith("${literal.") && value.endsWith("}"))
{
String literalValue = value.substring(10).replaceFirst("}$", "");
return (literalValue);
}
return (value);
}
/*******************************************************************************
** Setter for customEnvironment - protected - meant to be called (at least at this
** time), only in unit test
**
*******************************************************************************/
protected void setCustomEnvironment(Map<String, String> customEnvironment)
{
this.customEnvironment = customEnvironment;
}
/*******************************************************************************
**
*******************************************************************************/
private Map<String, String> getEnvironment()
{
if(this.customEnvironment != null)
{
return (this.customEnvironment);
}
return System.getenv();
}
}

16
src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessResult.java
View File

@ -36,7 +36,21 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
public class RunProcessResult extends AbstractQResult
{
private ProcessState processState;
private String error;
private String error;
/*******************************************************************************
**
*******************************************************************************/
@Override
public String toString()
{
return "RunProcessResult{error='" + error
+ ",records.size()=" + (processState == null ? null : processState.getRecords().size())
+ ",values=" + (processState == null ? null : processState.getValues())
+ "}";
}

89
src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QSecretReader.java
View File

@ -1,89 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata;
import java.util.Map;
/*******************************************************************************
** To avoid having secrets (passwords, access keys, etc) committed into meta data
** files, this class is used by the Enricher to "promote" values, such as ${env.ACCESS_KEY}
** to be read from the environment (or other secret providers (to be implemented)).
*******************************************************************************/
public class QSecretReader
{
private Map<String, String> customEnvironment;
/*******************************************************************************
** Translate a secret.
**
** If input is null, output is null.
** If input looks like ${env.X}, then the return value is the value of the env variable 'X'
** Else the output is the input.
*******************************************************************************/
public String readSecret(String value)
{
if(value == null)
{
return (null);
}
if(value.startsWith("${env.") && value.endsWith("}"))
{
String envVarName = value.substring(6).replaceFirst("}$", "");
String envValue = getEnvironment().get(envVarName);
return (envValue);
}
return (value);
}
/*******************************************************************************
** Setter for customEnvironment - protected - meant to be called (at least at this
** time), only in unit test
**
*******************************************************************************/
protected void setCustomEnvironment(Map<String, String> customEnvironment)
{
this.customEnvironment = customEnvironment;
}
/*******************************************************************************
**
*******************************************************************************/
private Map<String, String> getEnvironment()
{
if(this.customEnvironment != null)
{
return (this.customEnvironment);
}
return System.getenv();
}
}

42
src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -32,6 +33,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
@ -39,13 +42,22 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
*******************************************************************************/
public class BasicETLLoadFunction implements FunctionBody
{
private static final Logger LOG = LogManager.getLogger(BasicETLLoadFunction.class);
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
{
//////////////////////////////////////////////////////
// exit early with no-op if no records made it here //
//////////////////////////////////////////////////////
if(CollectionUtils.nullSafeIsEmpty(runFunctionRequest.getRecords()))
List<QRecord> inputRecords = runFunctionRequest.getRecords();
LOG.info("Received [" + inputRecords.size() + "] records to load");
if(CollectionUtils.nullSafeIsEmpty(inputRecords))
{
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, 0);
return;
@ -55,7 +67,7 @@ public class BasicETLLoadFunction implements FunctionBody
// put the destination table name in all records being inserted //
//////////////////////////////////////////////////////////////////
String table = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
for(QRecord record : runFunctionRequest.getRecords())
for(QRecord record : inputRecords)
{
record.setTableName(table);
}
@ -63,16 +75,26 @@ public class BasicETLLoadFunction implements FunctionBody
//////////////////////////////////////////
// run an insert request on the records //
//////////////////////////////////////////
InsertRequest insertRequest = new InsertRequest(runFunctionRequest.getInstance());
insertRequest.setSession(runFunctionRequest.getSession());
insertRequest.setTableName(table);
insertRequest.setRecords(runFunctionRequest.getRecords());
int recordsInserted = 0;
List<QRecord> outputRecords = new ArrayList<>();
int pageSize = 1000; // todo - make this a field?
InsertAction insertAction = new InsertAction();
InsertResult insertResult = insertAction.execute(insertRequest);
for(List<QRecord> page : CollectionUtils.getPages(inputRecords, pageSize))
{
LOG.info("Inserting a page of [" + page.size() + "] records. Progress: " + recordsInserted + " loaded out of " + inputRecords.size() + " total");
InsertRequest insertRequest = new InsertRequest(runFunctionRequest.getInstance());
insertRequest.setSession(runFunctionRequest.getSession());
insertRequest.setTableName(table);
insertRequest.setRecords(page);
runFunctionResult.setRecords(insertResult.getRecords());
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, insertResult.getRecords().size());
InsertAction insertAction = new InsertAction();
InsertResult insertResult = insertAction.execute(insertRequest);
outputRecords.addAll(insertResult.getRecords());
recordsInserted += insertResult.getRecords().size();
}
runFunctionResult.setRecords(outputRecords);
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, recordsInserted);
}
}

44
src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunction.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.adapters.JsonToQFieldMappingAdapter;
@ -33,9 +34,12 @@ import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFiel
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
@ -43,6 +47,8 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
*******************************************************************************/
public class BasicETLTransformFunction implements FunctionBody
{
private static final Logger LOG = LogManager.getLogger(BasicETLTransformFunction.class);
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
{
@ -72,11 +78,49 @@ public class BasicETLTransformFunction implements FunctionBody
String tableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
QTableMetaData table = runFunctionRequest.getInstance().getTable(tableName);
List<QRecord> mappedRecords = applyMapping(runFunctionRequest.getRecords(), table, keyBasedFieldMapping);
removeNonNumericValuesFromMappedRecords(table, mappedRecords);
runFunctionResult.setRecords(mappedRecords);
}
/*******************************************************************************
**
*******************************************************************************/
private void removeNonNumericValuesFromMappedRecords(QTableMetaData table, List<QRecord> records)
{
for(QRecord record : records)
{
for(QFieldMetaData field : table.getFields().values())
{
Object value = record.getValue(field.getName());
if(value != null && StringUtils.hasContent(String.valueOf(value)))
{
try
{
if(field.getType().equals(QFieldType.INTEGER))
{
Integer.parseInt(String.valueOf(value));
}
else if(field.getType().equals(QFieldType.DECIMAL))
{
new BigDecimal(String.valueOf(value));
}
}
catch(NumberFormatException nfe)
{
LOG.info("Removing non-numeric value [" + value + "] from field [" + field.getName() + "]");
record.setValue(field.getName(), null);
}
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/

281
src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java Normal file
View File

@ -0,0 +1,281 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.instances;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
/*******************************************************************************
** Unit test for QSecretReader
*******************************************************************************/
class QMetaDataVariableInterpreterTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
public void beforeEach()
{
System.setProperty("username", "joe");
System.setProperty("password", "b1d3n");
}
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
public void afterEach()
{
System.clearProperty("username");
System.clearProperty("password");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testInterpretObject() throws QException
{
GoodTestClass goodTestClass = new GoodTestClass();
goodTestClass.setUsername("${prop.username}");
goodTestClass.setPassword("${prop.password}");
new QMetaDataVariableInterpreter().interpretObject(goodTestClass);
assertEquals("joe", goodTestClass.getUsername());
assertEquals("b1d3n", goodTestClass.getPassword());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBadAnnotatedObjects()
{
assertThrows(QException.class, () -> new QMetaDataVariableInterpreter().interpretObject(new BadTestClassAnnotatedInteger()));
assertThrows(QException.class, () -> new QMetaDataVariableInterpreter().interpretObject(new BadTestClassNoGetter()));
assertThrows(QException.class, () -> new QMetaDataVariableInterpreter().interpretObject(new BadTestClassNoSetter()));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testInterpretFromEnvironment()
{
QMetaDataVariableInterpreter secretReader = new QMetaDataVariableInterpreter();
String key = "CUSTOM_PROPERTY";
String value = "ABCD-9876";
secretReader.setCustomEnvironment(Map.of(key, value));
assertNull(secretReader.interpret(null));
assertEquals("foo", secretReader.interpret("foo"));
assertNull(secretReader.interpret("${env.NOT-" + key + "}"));
assertEquals(value, secretReader.interpret("${env." + key + "}"));
assertEquals("${env.NOT-" + key, secretReader.interpret("${env.NOT-" + key));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testInterpretFromProperties()
{
QMetaDataVariableInterpreter secretReader = new QMetaDataVariableInterpreter();
String key = "MY_PROPERTY";
String value = "WXYZ-6789";
System.setProperty(key, value);
assertNull(secretReader.interpret(null));
assertEquals("foo", secretReader.interpret("foo"));
assertNull(secretReader.interpret("${prop.NOT-" + key + "}"));
assertEquals(value, secretReader.interpret("${prop." + key + "}"));
assertEquals("${prop.NOT-" + key, secretReader.interpret("${prop.NOT-" + key));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testInterpretLiterals()
{
QMetaDataVariableInterpreter secretReader = new QMetaDataVariableInterpreter();
assertEquals("${env.X}", secretReader.interpret("${literal.${env.X}}"));
assertEquals("${prop.X}", secretReader.interpret("${literal.${prop.X}}"));
assertEquals("${literal.X}", secretReader.interpret("${literal.${literal.X}}"));
}
/*******************************************************************************
**
*******************************************************************************/
@InterpretableFields(fieldNames = { "username", "password" })
public static class GoodTestClass
{
private String username;
private String password;
/*******************************************************************************
** Getter for username
**
*******************************************************************************/
public String getUsername()
{
return username;
}
/*******************************************************************************
** Setter for username
**
*******************************************************************************/
public void setUsername(String username)
{
this.username = username;
}
/*******************************************************************************
** Getter for password
**
*******************************************************************************/
public String getPassword()
{
return password;
}
/*******************************************************************************
** Setter for password
**
*******************************************************************************/
public void setPassword(String password)
{
this.password = password;
}
}
/*******************************************************************************
**
*******************************************************************************/
@InterpretableFields(fieldNames = { "port" })
public static class BadTestClassAnnotatedInteger
{
private Integer port;
/*******************************************************************************
** Getter for port
**
*******************************************************************************/
public Integer getPort()
{
return port;
}
/*******************************************************************************
** Setter for port
**
*******************************************************************************/
public void setPort(Integer port)
{
this.port = port;
}
}
/*******************************************************************************
**
*******************************************************************************/
@InterpretableFields(fieldNames = { "foo" })
public static class BadTestClassNoGetter
{
private String foo;
/*******************************************************************************
** Setter for foo
**
*******************************************************************************/
public void setFoo(String foo)
{
this.foo = foo;
}
}
/*******************************************************************************
**
*******************************************************************************/
@InterpretableFields(fieldNames = { "foo" })
public static class BadTestClassNoSetter
{
private String foo;
/*******************************************************************************
** Getter for foo
**
*******************************************************************************/
public String getFoo()
{
return foo;
}
}
}