mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-19 13:40:44 +00:00
QQQ-14 checkpoint, pre-demo
This commit is contained in:
@ -0,0 +1,40 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.instances;
|
||||
|
||||
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Class-level annotation to declare what fields should run through the variable
|
||||
** interpreter - e.g., to be replaced with env-var values at run-time.
|
||||
*******************************************************************************/
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target(ElementType.TYPE)
|
||||
public @interface InterpretableFields
|
||||
{
|
||||
String[] fieldNames() default {};
|
||||
}
|
||||
@ -0,0 +1,172 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.instances;
|
||||
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** To avoid having secrets (passwords, access keys, etc) committed into meta data
|
||||
** files, as well as to just let some meta data not be hard-coded, this class is
|
||||
** used by the Enricher to "promote" values, such as ${env.ACCESS_KEY}
|
||||
** to be read from the environment (or other secret providers (to be implemented)).
|
||||
**
|
||||
** Supported syntax / value sources are:
|
||||
** ${env.VAR} = system environment variables, e.g., export VAR=val
|
||||
** ${prop.VAR} = properties, e.g., -DVAR=val
|
||||
** ${literal.VAR} = get back a literal "VAR" (in case VAR matches some of the other supported syntax in here)
|
||||
*******************************************************************************/
|
||||
public class QMetaDataVariableInterpreter
|
||||
{
|
||||
private static final Logger LOG = LogManager.getLogger(QMetaDataVariableInterpreter.class);
|
||||
|
||||
private Map<String, String> customEnvironment;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void interpretObject(Object o) throws QException
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get the InterpretableFields from the object's class - exiting if the annotation isn't present //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
InterpretableFields interpretableFields = o.getClass().getAnnotation(InterpretableFields.class);
|
||||
if(interpretableFields == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// iterate over interpretable fields, interpreting each //
|
||||
//////////////////////////////////////////////////////////
|
||||
for(String fieldName : interpretableFields.fieldNames())
|
||||
{
|
||||
try
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
// get the getter & setter methods for the field (getMethod will throw if not found) //
|
||||
// enforce Strings-only at this time. //
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
String fieldNameUcFirst = fieldName.substring(0, 1).toUpperCase(Locale.ROOT) + fieldName.substring(1);
|
||||
Method getter = o.getClass().getMethod("get" + fieldNameUcFirst);
|
||||
Class<?> fieldType = getter.getReturnType();
|
||||
if(!fieldType.equals(String.class))
|
||||
{
|
||||
throw new QException("Interpretable field: " + fieldName + " on class " + o.getClass() + " is not a String (which is required at this time)");
|
||||
}
|
||||
Method setter = o.getClass().getMethod("set" + fieldNameUcFirst, fieldType);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// get the value - if it's null, move on, else, interpret it, and put it back in the object //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////
|
||||
Object value = getter.invoke(o);
|
||||
if(value == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
String interpreted = interpret((String) value);
|
||||
setter.invoke(o, interpreted);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
throw (new QException("Error interpreting variables in object " + o, e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Interpret a value string, which may be a variable, into its run-time value.
|
||||
**
|
||||
** If input is null, output is null.
|
||||
** If input looks like ${env.X}, then the return value is the value of the env variable 'X'
|
||||
** If input looks like ${prop.X}, then the return value is the value of the system property 'X'
|
||||
** If input looks like ${literal.X}, then the return value is the literal 'X'
|
||||
** - used if you really want to get back the literal value, ${env.X}, for example.
|
||||
** Else the output is the input.
|
||||
*******************************************************************************/
|
||||
public String interpret(String value)
|
||||
{
|
||||
if(value == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
if(value.startsWith("${env.") && value.endsWith("}"))
|
||||
{
|
||||
String envVarName = value.substring(6).replaceFirst("}$", "");
|
||||
String envValue = getEnvironment().get(envVarName);
|
||||
return (envValue);
|
||||
}
|
||||
|
||||
if(value.startsWith("${prop.") && value.endsWith("}"))
|
||||
{
|
||||
String propertyName = value.substring(7).replaceFirst("}$", "");
|
||||
String propertyValue = System.getProperty(propertyName);
|
||||
return (propertyValue);
|
||||
}
|
||||
|
||||
if(value.startsWith("${literal.") && value.endsWith("}"))
|
||||
{
|
||||
String literalValue = value.substring(10).replaceFirst("}$", "");
|
||||
return (literalValue);
|
||||
}
|
||||
|
||||
return (value);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for customEnvironment - protected - meant to be called (at least at this
|
||||
** time), only in unit test
|
||||
**
|
||||
*******************************************************************************/
|
||||
protected void setCustomEnvironment(Map<String, String> customEnvironment)
|
||||
{
|
||||
this.customEnvironment = customEnvironment;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private Map<String, String> getEnvironment()
|
||||
{
|
||||
if(this.customEnvironment != null)
|
||||
{
|
||||
return (this.customEnvironment);
|
||||
}
|
||||
|
||||
return System.getenv();
|
||||
}
|
||||
}
|
||||
@ -36,7 +36,21 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
public class RunProcessResult extends AbstractQResult
|
||||
{
|
||||
private ProcessState processState;
|
||||
private String error;
|
||||
private String error;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return "RunProcessResult{error='" + error
|
||||
+ ",records.size()=" + (processState == null ? null : processState.getRecords().size())
|
||||
+ ",values=" + (processState == null ? null : processState.getValues())
|
||||
+ "}";
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,89 +0,0 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.model.metadata;
|
||||
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** To avoid having secrets (passwords, access keys, etc) committed into meta data
|
||||
** files, this class is used by the Enricher to "promote" values, such as ${env.ACCESS_KEY}
|
||||
** to be read from the environment (or other secret providers (to be implemented)).
|
||||
*******************************************************************************/
|
||||
public class QSecretReader
|
||||
{
|
||||
private Map<String, String> customEnvironment;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Translate a secret.
|
||||
**
|
||||
** If input is null, output is null.
|
||||
** If input looks like ${env.X}, then the return value is the value of the env variable 'X'
|
||||
** Else the output is the input.
|
||||
*******************************************************************************/
|
||||
public String readSecret(String value)
|
||||
{
|
||||
if(value == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
if(value.startsWith("${env.") && value.endsWith("}"))
|
||||
{
|
||||
String envVarName = value.substring(6).replaceFirst("}$", "");
|
||||
String envValue = getEnvironment().get(envVarName);
|
||||
return (envValue);
|
||||
}
|
||||
|
||||
return (value);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for customEnvironment - protected - meant to be called (at least at this
|
||||
** time), only in unit test
|
||||
**
|
||||
*******************************************************************************/
|
||||
protected void setCustomEnvironment(Map<String, String> customEnvironment)
|
||||
{
|
||||
this.customEnvironment = customEnvironment;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private Map<String, String> getEnvironment()
|
||||
{
|
||||
if(this.customEnvironment != null)
|
||||
{
|
||||
return (this.customEnvironment);
|
||||
}
|
||||
|
||||
return System.getenv();
|
||||
}
|
||||
}
|
||||
@ -22,6 +22,7 @@
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
@ -32,6 +33,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -39,13 +42,22 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
*******************************************************************************/
|
||||
public class BasicETLLoadFunction implements FunctionBody
|
||||
{
|
||||
private static final Logger LOG = LogManager.getLogger(BasicETLLoadFunction.class);
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
|
||||
{
|
||||
//////////////////////////////////////////////////////
|
||||
// exit early with no-op if no records made it here //
|
||||
//////////////////////////////////////////////////////
|
||||
if(CollectionUtils.nullSafeIsEmpty(runFunctionRequest.getRecords()))
|
||||
List<QRecord> inputRecords = runFunctionRequest.getRecords();
|
||||
LOG.info("Received [" + inputRecords.size() + "] records to load");
|
||||
if(CollectionUtils.nullSafeIsEmpty(inputRecords))
|
||||
{
|
||||
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, 0);
|
||||
return;
|
||||
@ -55,7 +67,7 @@ public class BasicETLLoadFunction implements FunctionBody
|
||||
// put the destination table name in all records being inserted //
|
||||
//////////////////////////////////////////////////////////////////
|
||||
String table = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
|
||||
for(QRecord record : runFunctionRequest.getRecords())
|
||||
for(QRecord record : inputRecords)
|
||||
{
|
||||
record.setTableName(table);
|
||||
}
|
||||
@ -63,16 +75,26 @@ public class BasicETLLoadFunction implements FunctionBody
|
||||
//////////////////////////////////////////
|
||||
// run an insert request on the records //
|
||||
//////////////////////////////////////////
|
||||
InsertRequest insertRequest = new InsertRequest(runFunctionRequest.getInstance());
|
||||
insertRequest.setSession(runFunctionRequest.getSession());
|
||||
insertRequest.setTableName(table);
|
||||
insertRequest.setRecords(runFunctionRequest.getRecords());
|
||||
int recordsInserted = 0;
|
||||
List<QRecord> outputRecords = new ArrayList<>();
|
||||
int pageSize = 1000; // todo - make this a field?
|
||||
|
||||
InsertAction insertAction = new InsertAction();
|
||||
InsertResult insertResult = insertAction.execute(insertRequest);
|
||||
for(List<QRecord> page : CollectionUtils.getPages(inputRecords, pageSize))
|
||||
{
|
||||
LOG.info("Inserting a page of [" + page.size() + "] records. Progress: " + recordsInserted + " loaded out of " + inputRecords.size() + " total");
|
||||
InsertRequest insertRequest = new InsertRequest(runFunctionRequest.getInstance());
|
||||
insertRequest.setSession(runFunctionRequest.getSession());
|
||||
insertRequest.setTableName(table);
|
||||
insertRequest.setRecords(page);
|
||||
|
||||
runFunctionResult.setRecords(insertResult.getRecords());
|
||||
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, insertResult.getRecords().size());
|
||||
InsertAction insertAction = new InsertAction();
|
||||
InsertResult insertResult = insertAction.execute(insertRequest);
|
||||
outputRecords.addAll(insertResult.getRecords());
|
||||
|
||||
recordsInserted += insertResult.getRecords().size();
|
||||
}
|
||||
runFunctionResult.setRecords(outputRecords);
|
||||
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, recordsInserted);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.adapters.JsonToQFieldMappingAdapter;
|
||||
@ -33,9 +34,12 @@ import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFiel
|
||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -43,6 +47,8 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
*******************************************************************************/
|
||||
public class BasicETLTransformFunction implements FunctionBody
|
||||
{
|
||||
private static final Logger LOG = LogManager.getLogger(BasicETLTransformFunction.class);
|
||||
|
||||
@Override
|
||||
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
|
||||
{
|
||||
@ -72,11 +78,49 @@ public class BasicETLTransformFunction implements FunctionBody
|
||||
String tableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
|
||||
QTableMetaData table = runFunctionRequest.getInstance().getTable(tableName);
|
||||
List<QRecord> mappedRecords = applyMapping(runFunctionRequest.getRecords(), table, keyBasedFieldMapping);
|
||||
|
||||
removeNonNumericValuesFromMappedRecords(table, mappedRecords);
|
||||
|
||||
runFunctionResult.setRecords(mappedRecords);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void removeNonNumericValuesFromMappedRecords(QTableMetaData table, List<QRecord> records)
|
||||
{
|
||||
for(QRecord record : records)
|
||||
{
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
{
|
||||
Object value = record.getValue(field.getName());
|
||||
if(value != null && StringUtils.hasContent(String.valueOf(value)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if(field.getType().equals(QFieldType.INTEGER))
|
||||
{
|
||||
Integer.parseInt(String.valueOf(value));
|
||||
}
|
||||
else if(field.getType().equals(QFieldType.DECIMAL))
|
||||
{
|
||||
new BigDecimal(String.valueOf(value));
|
||||
}
|
||||
}
|
||||
catch(NumberFormatException nfe)
|
||||
{
|
||||
LOG.info("Removing non-numeric value [" + value + "] from field [" + field.getName() + "]");
|
||||
record.setValue(field.getName(), null);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
||||
Reference in New Issue
Block a user