mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 13:10:44 +00:00
QQQ-16 First iteration of basic ETL process
This commit is contained in:
@ -0,0 +1,73 @@
|
|||||||
|
/*
|
||||||
|
* QQQ - Low-code Application Framework for Engineers.
|
||||||
|
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||||
|
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||||
|
* contact@kingsrook.com
|
||||||
|
* https://github.com/Kingsrook/
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||||
|
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import com.kingsrook.qqq.backend.core.callbacks.QProcessCallback;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.query.QQueryFilter;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Provide callback functionality for the BasicETL process
|
||||||
|
*******************************************************************************/
|
||||||
|
public class BasicETLCallback implements QProcessCallback
|
||||||
|
{
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Get the filter query for this callback.
|
||||||
|
*******************************************************************************/
|
||||||
|
@Override
|
||||||
|
public QQueryFilter getQueryFilter()
|
||||||
|
{
|
||||||
|
// todo - possibly get something from params? through state? added as a method arg?
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Get the field values for this callback.
|
||||||
|
*******************************************************************************/
|
||||||
|
@SuppressWarnings("checkstyle:Indentation")
|
||||||
|
@Override
|
||||||
|
public Map<String, Serializable> getFieldValues(List<QFieldMetaData> fields)
|
||||||
|
{
|
||||||
|
Map<String, Serializable> rs = new HashMap<>();
|
||||||
|
for(QFieldMetaData field : fields)
|
||||||
|
{
|
||||||
|
// TODO - replace this whole thing with our params mechanism
|
||||||
|
// TODO - add default methods to the interface that throw, presumably?
|
||||||
|
rs.put(field.getName(), switch(field.getName())
|
||||||
|
{
|
||||||
|
case BasicETLProcess.FIELD_SOURCE_TABLE -> "personFile";
|
||||||
|
case BasicETLProcess.FIELD_DESTINATION_TABLE -> "person";
|
||||||
|
default -> throw new IllegalArgumentException("Unhandled field: " + field.getName());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return (rs);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,59 @@
|
|||||||
|
/*
|
||||||
|
* QQQ - Low-code Application Framework for Engineers.
|
||||||
|
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||||
|
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||||
|
* contact@kingsrook.com
|
||||||
|
* https://github.com/Kingsrook/
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||||
|
|
||||||
|
|
||||||
|
import com.kingsrook.qqq.backend.core.actions.QueryAction;
|
||||||
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
|
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Function body for performing the Extract step of a basic ETL process.
|
||||||
|
*******************************************************************************/
|
||||||
|
public class BasicETLExtractFunction implements FunctionBody
|
||||||
|
{
|
||||||
|
@Override
|
||||||
|
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
|
||||||
|
{
|
||||||
|
QueryRequest queryRequest = new QueryRequest(runFunctionRequest.getInstance());
|
||||||
|
queryRequest.setSession(runFunctionRequest.getSession());
|
||||||
|
queryRequest.setTableName(runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE));
|
||||||
|
// queryRequest.setSkip(integerQueryParam(context, "skip"));
|
||||||
|
// queryRequest.setLimit(integerQueryParam(context, "limit"));
|
||||||
|
|
||||||
|
// todo? String filter = stringQueryParam(context, "filter");
|
||||||
|
// if(filter != null)
|
||||||
|
// {
|
||||||
|
// queryRequest.setFilter(JsonUtils.toObject(filter, QQueryFilter.class));
|
||||||
|
// }
|
||||||
|
|
||||||
|
QueryAction queryAction = new QueryAction();
|
||||||
|
QueryResult queryResult = queryAction.execute(queryRequest);
|
||||||
|
|
||||||
|
runFunctionResult.setRecords(queryResult.getRecords());
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,78 @@
|
|||||||
|
/*
|
||||||
|
* QQQ - Low-code Application Framework for Engineers.
|
||||||
|
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||||
|
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||||
|
* contact@kingsrook.com
|
||||||
|
* https://github.com/Kingsrook/
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||||
|
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import com.kingsrook.qqq.backend.core.actions.InsertAction;
|
||||||
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
|
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||||
|
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Function body for performing the Load step of a basic ETL process.
|
||||||
|
*******************************************************************************/
|
||||||
|
public class BasicETLLoadFunction implements FunctionBody
|
||||||
|
{
|
||||||
|
@Override
|
||||||
|
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
|
||||||
|
{
|
||||||
|
//////////////////////////////////////////////////////
|
||||||
|
// exit early with no-op if no records made it here //
|
||||||
|
//////////////////////////////////////////////////////
|
||||||
|
if(CollectionUtils.nullSafeIsEmpty(runFunctionRequest.getRecords()))
|
||||||
|
{
|
||||||
|
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, 0);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
//////////////////////////////////////////////////////////////////
|
||||||
|
// put the destination table name in all records being inserted //
|
||||||
|
//////////////////////////////////////////////////////////////////
|
||||||
|
String table = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
|
||||||
|
for(QRecord record : runFunctionRequest.getRecords())
|
||||||
|
{
|
||||||
|
record.setTableName(table);
|
||||||
|
}
|
||||||
|
|
||||||
|
//////////////////////////////////////////
|
||||||
|
// run an insert request on the records //
|
||||||
|
//////////////////////////////////////////
|
||||||
|
InsertRequest insertRequest = new InsertRequest(runFunctionRequest.getInstance());
|
||||||
|
insertRequest.setSession(runFunctionRequest.getSession());
|
||||||
|
insertRequest.setTableName(table);
|
||||||
|
insertRequest.setRecords(runFunctionRequest.getRecords());
|
||||||
|
|
||||||
|
InsertAction insertAction = new InsertAction();
|
||||||
|
InsertResult insertResult = insertAction.execute(insertRequest);
|
||||||
|
|
||||||
|
runFunctionResult.setRecords(insertResult.getRecords());
|
||||||
|
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, insertResult.getRecords().size());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,77 @@
|
|||||||
|
/*
|
||||||
|
* QQQ - Low-code Application Framework for Engineers.
|
||||||
|
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||||
|
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||||
|
* contact@kingsrook.com
|
||||||
|
* https://github.com/Kingsrook/
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||||
|
|
||||||
|
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Definition for Basic ETL process.
|
||||||
|
*******************************************************************************/
|
||||||
|
public class BasicETLProcess
|
||||||
|
{
|
||||||
|
public static final String FIELD_SOURCE_TABLE = "sourceTable";
|
||||||
|
public static final String FIELD_DESTINATION_TABLE = "destinationTable";
|
||||||
|
public static final String FIELD_RECORD_COUNT = "recordCount";
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public QProcessMetaData defineProcessMetaData()
|
||||||
|
{
|
||||||
|
QFunctionMetaData extractFunction = new QFunctionMetaData()
|
||||||
|
.withName("extract")
|
||||||
|
.withCode(new QCodeReference()
|
||||||
|
.withName(BasicETLExtractFunction.class.getName())
|
||||||
|
.withCodeType(QCodeType.JAVA)
|
||||||
|
.withCodeUsage(QCodeUsage.FUNCTION))
|
||||||
|
.withInputData(new QFunctionInputMetaData()
|
||||||
|
.addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)));
|
||||||
|
|
||||||
|
QFunctionMetaData loadFunction = new QFunctionMetaData()
|
||||||
|
.withName("load")
|
||||||
|
.withCode(new QCodeReference()
|
||||||
|
.withName(BasicETLLoadFunction.class.getName())
|
||||||
|
.withCodeType(QCodeType.JAVA)
|
||||||
|
.withCodeUsage(QCodeUsage.FUNCTION))
|
||||||
|
.withInputData(new QFunctionInputMetaData()
|
||||||
|
.addField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)))
|
||||||
|
.withOutputMetaData(new QFunctionOutputMetaData()
|
||||||
|
.addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER)));
|
||||||
|
|
||||||
|
return new QProcessMetaData()
|
||||||
|
.withName("etl.basic")
|
||||||
|
.addFunction(extractFunction)
|
||||||
|
.addFunction(loadFunction);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,94 @@
|
|||||||
|
/*
|
||||||
|
* QQQ - Low-code Application Framework for Engineers.
|
||||||
|
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||||
|
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||||
|
* contact@kingsrook.com
|
||||||
|
* https://github.com/Kingsrook/
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||||
|
|
||||||
|
|
||||||
|
import com.kingsrook.qqq.backend.core.actions.RunProcessAction;
|
||||||
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessRequest;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessResult;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Unit test for BasicETLProcess
|
||||||
|
*******************************************************************************/
|
||||||
|
class BasicETLProcessTest
|
||||||
|
{
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
public void test() throws QException
|
||||||
|
{
|
||||||
|
BasicETLProcess basicETLProcess = new BasicETLProcess();
|
||||||
|
QProcessMetaData processMetaData = basicETLProcess.defineProcessMetaData();
|
||||||
|
QInstance instance = TestUtils.defineInstance();
|
||||||
|
RunProcessRequest request = new RunProcessRequest(instance);
|
||||||
|
|
||||||
|
instance.addProcess(processMetaData);
|
||||||
|
defineFileBackendAndPersonFileTable(instance);
|
||||||
|
|
||||||
|
request.setSession(TestUtils.getMockSession());
|
||||||
|
request.setProcessName(processMetaData.getName());
|
||||||
|
request.setCallback(new BasicETLCallback()); // todo - uh, maybe a method on the process to get its callback?
|
||||||
|
RunProcessResult result = new RunProcessAction().execute(request);
|
||||||
|
assertNotNull(result);
|
||||||
|
assertNull(result.getError());
|
||||||
|
assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Define the 'person' table used in standard tests.
|
||||||
|
*******************************************************************************/
|
||||||
|
public static void defineFileBackendAndPersonFileTable(QInstance instance)
|
||||||
|
{
|
||||||
|
QTableMetaData personFileTable = new QTableMetaData()
|
||||||
|
.withName("personFile")
|
||||||
|
.withLabel("Person File")
|
||||||
|
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
|
||||||
|
.withPrimaryKeyField("id")
|
||||||
|
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||||
|
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
|
||||||
|
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))
|
||||||
|
.withField(new QFieldMetaData("firstName", QFieldType.STRING))
|
||||||
|
.withField(new QFieldMetaData("lastName", QFieldType.STRING))
|
||||||
|
.withField(new QFieldMetaData("birthDate", QFieldType.DATE))
|
||||||
|
.withField(new QFieldMetaData("email", QFieldType.STRING))
|
||||||
|
.withField(new QFieldMetaData("homeState", QFieldType.STRING).withPossibleValueSourceName("state"));
|
||||||
|
|
||||||
|
instance.addTable(personFileTable);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user