diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLCallback.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLCallback.java new file mode 100644 index 00000000..e9f14ba7 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLCallback.java @@ -0,0 +1,73 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.callbacks.QProcessCallback; +import com.kingsrook.qqq.backend.core.model.actions.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; + + +/******************************************************************************* + ** Provide callback functionality for the BasicETL process + *******************************************************************************/ +public class BasicETLCallback implements QProcessCallback +{ + + /******************************************************************************* + ** Get the filter query for this callback. + *******************************************************************************/ + @Override + public QQueryFilter getQueryFilter() + { + // todo - possibly get something from params? through state? added as a method arg? + return null; + } + + + + /******************************************************************************* + ** Get the field values for this callback. + *******************************************************************************/ + @SuppressWarnings("checkstyle:Indentation") + @Override + public Map getFieldValues(List fields) + { + Map rs = new HashMap<>(); + for(QFieldMetaData field : fields) + { + // TODO - replace this whole thing with our params mechanism + // TODO - add default methods to the interface that throw, presumably? + rs.put(field.getName(), switch(field.getName()) + { + case BasicETLProcess.FIELD_SOURCE_TABLE -> "personFile"; + case BasicETLProcess.FIELD_DESTINATION_TABLE -> "person"; + default -> throw new IllegalArgumentException("Unhandled field: " + field.getName()); + }); + } + return (rs); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java new file mode 100644 index 00000000..cb2e680d --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLExtractFunction.java @@ -0,0 +1,59 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; + + +import com.kingsrook.qqq.backend.core.actions.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult; + + +/******************************************************************************* + ** Function body for performing the Extract step of a basic ETL process. + *******************************************************************************/ +public class BasicETLExtractFunction implements FunctionBody +{ + @Override + public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + { + QueryRequest queryRequest = new QueryRequest(runFunctionRequest.getInstance()); + queryRequest.setSession(runFunctionRequest.getSession()); + queryRequest.setTableName(runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE)); + // queryRequest.setSkip(integerQueryParam(context, "skip")); + // queryRequest.setLimit(integerQueryParam(context, "limit")); + + // todo? String filter = stringQueryParam(context, "filter"); + // if(filter != null) + // { + // queryRequest.setFilter(JsonUtils.toObject(filter, QQueryFilter.class)); + // } + + QueryAction queryAction = new QueryAction(); + QueryResult queryResult = queryAction.execute(queryRequest); + + runFunctionResult.setRecords(queryResult.getRecords()); + } +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java new file mode 100644 index 00000000..23380e56 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadFunction.java @@ -0,0 +1,78 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.InsertAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; +import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** Function body for performing the Load step of a basic ETL process. + *******************************************************************************/ +public class BasicETLLoadFunction implements FunctionBody +{ + @Override + public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException + { + ////////////////////////////////////////////////////// + // exit early with no-op if no records made it here // + ////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeIsEmpty(runFunctionRequest.getRecords())) + { + runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, 0); + return; + } + + ////////////////////////////////////////////////////////////////// + // put the destination table name in all records being inserted // + ////////////////////////////////////////////////////////////////// + String table = runFunctionRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE); + for(QRecord record : runFunctionRequest.getRecords()) + { + record.setTableName(table); + } + + ////////////////////////////////////////// + // run an insert request on the records // + ////////////////////////////////////////// + InsertRequest insertRequest = new InsertRequest(runFunctionRequest.getInstance()); + insertRequest.setSession(runFunctionRequest.getSession()); + insertRequest.setTableName(table); + insertRequest.setRecords(runFunctionRequest.getRecords()); + + InsertAction insertAction = new InsertAction(); + InsertResult insertResult = insertAction.execute(insertRequest); + + runFunctionResult.setRecords(insertResult.getRecords()); + runFunctionResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, insertResult.getRecords().size()); + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java new file mode 100644 index 00000000..c6dd0287 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; + + +import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; + + +/******************************************************************************* + ** Definition for Basic ETL process. + *******************************************************************************/ +public class BasicETLProcess +{ + public static final String FIELD_SOURCE_TABLE = "sourceTable"; + public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + public static final String FIELD_RECORD_COUNT = "recordCount"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QProcessMetaData defineProcessMetaData() + { + QFunctionMetaData extractFunction = new QFunctionMetaData() + .withName("extract") + .withCode(new QCodeReference() + .withName(BasicETLExtractFunction.class.getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.FUNCTION)) + .withInputData(new QFunctionInputMetaData() + .addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING))); + + QFunctionMetaData loadFunction = new QFunctionMetaData() + .withName("load") + .withCode(new QCodeReference() + .withName(BasicETLLoadFunction.class.getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.FUNCTION)) + .withInputData(new QFunctionInputMetaData() + .addField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING))) + .withOutputMetaData(new QFunctionOutputMetaData() + .addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER))); + + return new QProcessMetaData() + .withName("etl.basic") + .addFunction(extractFunction) + .addFunction(loadFunction); + } +} diff --git a/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java new file mode 100644 index 00000000..cfb5fe9b --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java @@ -0,0 +1,94 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; + + +import com.kingsrook.qqq.backend.core.actions.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessResult; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for BasicETLProcess + *******************************************************************************/ +class BasicETLProcessTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws QException + { + BasicETLProcess basicETLProcess = new BasicETLProcess(); + QProcessMetaData processMetaData = basicETLProcess.defineProcessMetaData(); + QInstance instance = TestUtils.defineInstance(); + RunProcessRequest request = new RunProcessRequest(instance); + + instance.addProcess(processMetaData); + defineFileBackendAndPersonFileTable(instance); + + request.setSession(TestUtils.getMockSession()); + request.setProcessName(processMetaData.getName()); + request.setCallback(new BasicETLCallback()); // todo - uh, maybe a method on the process to get its callback? + RunProcessResult result = new RunProcessAction().execute(request); + assertNotNull(result); + assertNull(result.getError()); + assertTrue(result.getRecords().stream().allMatch(r -> r.getValues().containsKey("id")), "records should have an id, set by the process"); + } + + + + /******************************************************************************* + ** Define the 'person' table used in standard tests. + *******************************************************************************/ + public static void defineFileBackendAndPersonFileTable(QInstance instance) + { + QTableMetaData personFileTable = new QTableMetaData() + .withName("personFile") + .withLabel("Person File") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("firstName", QFieldType.STRING)) + .withField(new QFieldMetaData("lastName", QFieldType.STRING)) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE)) + .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withField(new QFieldMetaData("homeState", QFieldType.STRING).withPossibleValueSourceName("state")); + + instance.addTable(personFileTable); + } + +} \ No newline at end of file