mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 13:10:44 +00:00
QQQ-16 First iteration of basic ETL process
This commit is contained in:
@ -0,0 +1,73 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.callbacks.QProcessCallback;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Provide callback functionality for the BasicETL process
|
||||
*******************************************************************************/
|
||||
public class BasicETLCallback implements QProcessCallback
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** Get the filter query for this callback.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public QQueryFilter getQueryFilter()
|
||||
{
|
||||
// todo - possibly get something from params? through state? added as a method arg?
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Get the field values for this callback.
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("checkstyle:Indentation")
|
||||
@Override
|
||||
public Map<String, Serializable> getFieldValues(List<QFieldMetaData> fields)
|
||||
{
|
||||
Map<String, Serializable> rs = new HashMap<>();
|
||||
for(QFieldMetaData field : fields)
|
||||
{
|
||||
// TODO - replace this whole thing with our params mechanism
|
||||
// TODO - add default methods to the interface that throw, presumably?
|
||||
rs.put(field.getName(), switch(field.getName())
|
||||
{
|
||||
case BasicETLProcess.FIELD_SOURCE_TABLE -> "personFile";
|
||||
case BasicETLProcess.FIELD_DESTINATION_TABLE -> "person";
|
||||
default -> throw new IllegalArgumentException("Unhandled field: " + field.getName());
|
||||
});
|
||||
}
|
||||
return (rs);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,59 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.actions.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Function body for performing the Extract step of a basic ETL process.
|
||||
*******************************************************************************/
|
||||
public class BasicETLExtractFunction implements FunctionBody
|
||||
{
|
||||
@Override
|
||||
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
|
||||
{
|
||||
QueryRequest queryRequest = new QueryRequest(runFunctionRequest.getInstance());
|
||||
queryRequest.setSession(runFunctionRequest.getSession());
|
||||
queryRequest.setTableName(runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE));
|
||||
// queryRequest.setSkip(integerQueryParam(context, "skip"));
|
||||
// queryRequest.setLimit(integerQueryParam(context, "limit"));
|
||||
|
||||
// todo? String filter = stringQueryParam(context, "filter");
|
||||
// if(filter != null)
|
||||
// {
|
||||
// queryRequest.setFilter(JsonUtils.toObject(filter, QQueryFilter.class));
|
||||
// }
|
||||
|
||||
QueryAction queryAction = new QueryAction();
|
||||
QueryResult queryResult = queryAction.execute(queryRequest);
|
||||
|
||||
runFunctionResult.setRecords(queryResult.getRecords());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,78 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Function body for performing the Load step of a basic ETL process.
|
||||
*******************************************************************************/
|
||||
public class BasicETLLoadFunction implements FunctionBody
|
||||
{
|
||||
@Override
|
||||
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
|
||||
{
|
||||
//////////////////////////////////////////////////////
|
||||
// exit early with no-op if no records made it here //
|
||||
//////////////////////////////////////////////////////
|
||||
if(CollectionUtils.nullSafeIsEmpty(runFunctionRequest.getRecords()))
|
||||
{
|
||||
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, 0);
|
||||
return;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// put the destination table name in all records being inserted //
|
||||
//////////////////////////////////////////////////////////////////
|
||||
String table = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
|
||||
for(QRecord record : runFunctionRequest.getRecords())
|
||||
{
|
||||
record.setTableName(table);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////
|
||||
// run an insert request on the records //
|
||||
//////////////////////////////////////////
|
||||
InsertRequest insertRequest = new InsertRequest(runFunctionRequest.getInstance());
|
||||
insertRequest.setSession(runFunctionRequest.getSession());
|
||||
insertRequest.setTableName(table);
|
||||
insertRequest.setRecords(runFunctionRequest.getRecords());
|
||||
|
||||
InsertAction insertAction = new InsertAction();
|
||||
InsertResult insertResult = insertAction.execute(insertRequest);
|
||||
|
||||
runFunctionResult.setRecords(insertResult.getRecords());
|
||||
runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, insertResult.getRecords().size());
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,77 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Definition for Basic ETL process.
|
||||
*******************************************************************************/
|
||||
public class BasicETLProcess
|
||||
{
|
||||
public static final String FIELD_SOURCE_TABLE = "sourceTable";
|
||||
public static final String FIELD_DESTINATION_TABLE = "destinationTable";
|
||||
public static final String FIELD_RECORD_COUNT = "recordCount";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public QProcessMetaData defineProcessMetaData()
|
||||
{
|
||||
QFunctionMetaData extractFunction = new QFunctionMetaData()
|
||||
.withName("extract")
|
||||
.withCode(new QCodeReference()
|
||||
.withName(BasicETLExtractFunction.class.getName())
|
||||
.withCodeType(QCodeType.JAVA)
|
||||
.withCodeUsage(QCodeUsage.FUNCTION))
|
||||
.withInputData(new QFunctionInputMetaData()
|
||||
.addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING)));
|
||||
|
||||
QFunctionMetaData loadFunction = new QFunctionMetaData()
|
||||
.withName("load")
|
||||
.withCode(new QCodeReference()
|
||||
.withName(BasicETLLoadFunction.class.getName())
|
||||
.withCodeType(QCodeType.JAVA)
|
||||
.withCodeUsage(QCodeUsage.FUNCTION))
|
||||
.withInputData(new QFunctionInputMetaData()
|
||||
.addField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING)))
|
||||
.withOutputMetaData(new QFunctionOutputMetaData()
|
||||
.addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER)));
|
||||
|
||||
return new QProcessMetaData()
|
||||
.withName("etl.basic")
|
||||
.addFunction(extractFunction)
|
||||
.addFunction(loadFunction);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,94 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.actions.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessResult;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for BasicETLProcess
|
||||
*******************************************************************************/
|
||||
class BasicETLProcessTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test() throws QException
|
||||
{
|
||||
BasicETLProcess basicETLProcess = new BasicETLProcess();
|
||||
QProcessMetaData processMetaData = basicETLProcess.defineProcessMetaData();
|
||||
QInstance instance = TestUtils.defineInstance();
|
||||
RunProcessRequest request = new RunProcessRequest(instance);
|
||||
|
||||
instance.addProcess(processMetaData);
|
||||
defineFileBackendAndPersonFileTable(instance);
|
||||
|
||||
request.setSession(TestUtils.getMockSession());
|
||||
request.setProcessName(processMetaData.getName());
|
||||
request.setCallback(new BasicETLCallback()); // todo - uh, maybe a method on the process to get its callback?
|
||||
RunProcessResult result = new RunProcessAction().execute(request);
|
||||
assertNotNull(result);
|
||||
assertNull(result.getError());
|
||||
assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Define the 'person' table used in standard tests.
|
||||
*******************************************************************************/
|
||||
public static void defineFileBackendAndPersonFileTable(QInstance instance)
|
||||
{
|
||||
QTableMetaData personFileTable = new QTableMetaData()
|
||||
.withName("personFile")
|
||||
.withLabel("Person File")
|
||||
.withBackendName(TestUtils.DEFAULT_BACKEND_NAME)
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))
|
||||
.withField(new QFieldMetaData("firstName", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("lastName", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("birthDate", QFieldType.DATE))
|
||||
.withField(new QFieldMetaData("email", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("homeState", QFieldType.STRING).withPossibleValueSourceName("state"));
|
||||
|
||||
instance.addTable(personFileTable);
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user