From be74eacb13a5960361e55e9c63c825508d551667 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 24 Jun 2022 16:05:34 -0500 Subject: [PATCH] QQQ-14 feedback from code review --- .../core/actions/RunFunctionAction.java | 25 +++-- .../core/actions/RunProcessAction.java | 8 +- .../backend/core/callbacks/NoopCallback.java | 58 ----------- .../backend/core/interfaces/FunctionBody.java | 9 +- .../core/model/metadata/QBackendMetaData.java | 16 +-- .../core/model/metadata/QInstance.java | 9 +- .../serialization/DeserializerUtils.java | 99 +++++++++++++------ .../etl/basic/BasicETLCallback.java | 73 -------------- .../etl/basic/BasicETLProcess.java | 3 +- .../core/state/InMemoryStateProvider.java | 9 +- .../core/state/StateProviderInterface.java | 14 ++- .../core/state/TempFileStateProvider.java | 11 ++- .../etl/basic/BasicETLProcessTest.java | 43 +------- .../core/state/InMemoryStateProviderTest.java | 4 +- .../core/state/TempFileStateProviderTest.java | 4 +- .../qqq/backend/core/utils/TestUtils.java | 25 ++++- 16 files changed, 164 insertions(+), 246 deletions(-) delete mode 100644 src/main/java/com/kingsrook/qqq/backend/core/callbacks/NoopCallback.java delete mode 100644 src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLCallback.java diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/RunFunctionAction.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/RunFunctionAction.java index d91e2238..326d8f7a 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/actions/RunFunctionAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/RunFunctionAction.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.callbacks.QProcessCallback; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.interfaces.FunctionBody; import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; @@ -54,9 +55,6 @@ public class RunFunctionAction { ActionHelper.validateSession(runFunctionRequest); - /////////////////////////////////////////////////////// - // - /////////////////////////////////////////////////////// QProcessMetaData process = runFunctionRequest.getInstance().getProcess(runFunctionRequest.getProcessName()); if(process == null) { @@ -88,10 +86,10 @@ public class RunFunctionAction ** via the callback ** *******************************************************************************/ - private void ensureInputFieldsAreInRequest(RunFunctionRequest runFunctionRequest, QFunctionMetaData function) + private void ensureInputFieldsAreInRequest(RunFunctionRequest runFunctionRequest, QFunctionMetaData function) throws QException { QFunctionInputMetaData inputMetaData = function.getInputMetaData(); - if (inputMetaData == null) + if(inputMetaData == null) { return; } @@ -109,7 +107,13 @@ public class RunFunctionAction if(!fieldsToGet.isEmpty()) { - Map fieldValues = runFunctionRequest.getCallback().getFieldValues(fieldsToGet); + QProcessCallback callback = runFunctionRequest.getCallback(); + if(callback == null) + { + throw (new QException("Function is missing values for fields, but no callback was present to request fields from a user")); + } + + Map fieldValues = callback.getFieldValues(fieldsToGet); for(Map.Entry entry : fieldValues.entrySet()) { runFunctionRequest.addValue(entry.getKey(), entry.getValue()); @@ -138,7 +142,14 @@ public class RunFunctionAction // todo - handle this being async (e.g., http) // seems like it just needs to throw, breaking this flow, and to send a response to the frontend, directing it to prompt the user for the needed data // then this function can re-run, hopefully with the needed data. - queryRequest.setFilter(runFunctionRequest.getCallback().getQueryFilter()); + + QProcessCallback callback = runFunctionRequest.getCallback(); + if(callback == null) + { + throw (new QException("Function is missing input records, but no callback was present to get a query filter from a user")); + } + + queryRequest.setFilter(callback.getQueryFilter()); QueryResult queryResult = new QueryAction().execute(queryRequest); runFunctionRequest.setRecords(queryResult.getRecords()); diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/RunProcessAction.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/RunProcessAction.java index a3a595b3..5bb9ec18 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/actions/RunProcessAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/actions/RunProcessAction.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions; import java.util.List; +import java.util.Optional; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState; import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest; @@ -145,10 +146,11 @@ public class RunProcessAction ** Load the process state into a function request from the state provider ** *******************************************************************************/ - private void loadState(UUIDStateKey stateKey, RunFunctionRequest runFunctionRequest) + private void loadState(UUIDStateKey stateKey, RunFunctionRequest runFunctionRequest) throws QException { - ProcessState processState = getStateProvider().get(ProcessState.class, stateKey); - runFunctionRequest.seedFromProcessState(processState); + Optional processState = getStateProvider().get(ProcessState.class, stateKey); + runFunctionRequest.seedFromProcessState(processState + .orElseThrow(() -> new QException("Could not find process state in state provider."))); } } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/callbacks/NoopCallback.java b/src/main/java/com/kingsrook/qqq/backend/core/callbacks/NoopCallback.java deleted file mode 100644 index 4c3fa2c5..00000000 --- a/src/main/java/com/kingsrook/qqq/backend/core/callbacks/NoopCallback.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.callbacks; - - -import java.io.Serializable; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import com.kingsrook.qqq.backend.core.model.actions.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; - - -/******************************************************************************* - ** Simple implementation of a callback, that does no-op (returns empty objects). - ** Useful for scaffolding, perhaps tests. - *******************************************************************************/ -public class NoopCallback implements QProcessCallback -{ - /******************************************************************************* - ** Get the filter query for this callback. - *******************************************************************************/ - @Override - public QQueryFilter getQueryFilter() - { - return (new QQueryFilter()); - } - - - - /******************************************************************************* - ** Get the field values for this callback. - *******************************************************************************/ - @Override - public Map getFieldValues(List fields) - { - return (Collections.emptyMap()); - } -} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/interfaces/FunctionBody.java b/src/main/java/com/kingsrook/qqq/backend/core/interfaces/FunctionBody.java index ab720fe3..12310ee5 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/interfaces/FunctionBody.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/interfaces/FunctionBody.java @@ -28,12 +28,17 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult; /******************************************************************************* - ** TODO - document! + ** Simple interface that a "custom" function (as in, a component of a Process) + ** must implement. *******************************************************************************/ public interface FunctionBody { /******************************************************************************* - ** TODO - document! + ** Execute the function - using the request as input, and the result as output. + ** + ** TODO - think about - why take the Result object as a param, instead of return it? + ** Is this way easier for inter-language operability maybe? + * Also - there's way too much "process-specific gunk" in the Request object - can we simplify it? *******************************************************************************/ void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException; } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java index 6e585512..24d0721e 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java @@ -42,6 +42,7 @@ public class QBackendMetaData // @JsonFilter("secretsFilter") + /******************************************************************************* ** Default Constructor. *******************************************************************************/ @@ -51,17 +52,6 @@ public class QBackendMetaData - /******************************************************************************* - ** Copy Constructor. Meant for use by sub-classes. Should copy all fields! - *******************************************************************************/ - protected QBackendMetaData(QBackendMetaData source) - { - this.name = source.name; - this.backendType = source.backendType; - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -83,7 +73,7 @@ public class QBackendMetaData /******************************************************************************* - ** + ** Fluent setter, returning generically, to help sub-class fluent flows *******************************************************************************/ @SuppressWarnings("unchecked") public T withName(String name) @@ -136,7 +126,7 @@ public class QBackendMetaData /******************************************************************************* - ** + ** Fluent setter, returning generically, to help sub-class fluent flows *******************************************************************************/ @SuppressWarnings("unchecked") public T withBackendType(String backendType) diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 39510b15..04e0ba4d 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -59,7 +59,7 @@ public class QInstance /******************************************************************************* - ** + ** Get the backend for a given table name *******************************************************************************/ public QBackendMetaData getBackendForTable(String tableName) { @@ -69,19 +69,16 @@ public class QInstance throw (new IllegalArgumentException("No table with name [" + tableName + "] found in this instance.")); } - QBackendMetaData backend = backends.get(table.getBackendName()); - ////////////////////////////////////////////////////////////////////////////////////////////// // validation should already let us know that this is valid, so no need to check/throw here // ////////////////////////////////////////////////////////////////////////////////////////////// - - return (backend); + return (backends.get(table.getBackendName())); } /******************************************************************************* - ** + ** Get the list of processes associated with a given table name *******************************************************************************/ public List getProcessesForTable(String tableName) { diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/DeserializerUtils.java b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/DeserializerUtils.java index 19dd795c..4a52ed3e 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/DeserializerUtils.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/DeserializerUtils.java @@ -38,19 +38,30 @@ import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException; import com.kingsrook.qqq.backend.core.modules.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* - ** + ** Utility methods to help with deserializing JSON streams into QQQ models. + ** Specifically meant to be used within a jackson custom deserializer (e.g., + ** an implementation of JsonDeserializer). *******************************************************************************/ public class DeserializerUtils { + private static final Logger LOG = LogManager.getLogger(DeserializerUtils.class); + + /******************************************************************************* - ** + ** For a given (jackson, JSON) treeNode, look at its backendType property, + ** and return an instance of the corresponding QBackendModule. *******************************************************************************/ public static QBackendModuleInterface getBackendModule(TreeNode treeNode) throws IOException { + ///////////////////////////////////////////////////////////////////////////////// + // validate the backendType property is present, as text, in the json treeNode // + ///////////////////////////////////////////////////////////////////////////////// TreeNode backendTypeTreeNode = treeNode.get("backendType"); if(backendTypeTreeNode == null || backendTypeTreeNode instanceof NullNode) { @@ -61,44 +72,67 @@ public class DeserializerUtils { throw new IOException("backendType is not a string value (is: " + backendTypeTreeNode.getClass().getSimpleName() + ")"); } - else - { - String backendType = textNode.asText(); - try - { - return new QBackendModuleDispatcher().getQBackendModule(backendType); - } - catch(QModuleDispatchException e) - { - throw (new IOException(e)); - } + try + { + ///////////////////////////////////////////////////////////////////////////////////////////////// + // get the value of the backendType json node, and use it to look up the qBackendModule object // + ///////////////////////////////////////////////////////////////////////////////////////////////// + String backendType = textNode.asText(); + return new QBackendModuleDispatcher().getQBackendModule(backendType); + } + catch(QModuleDispatchException e) + { + throw (new IOException(e)); } } /******************************************************************************* + ** Using reflection, create & populate an instance of a class, based on the + ** properties in a jackson/json treeNode. ** *******************************************************************************/ public static T reflectivelyDeserialize(Class outputClass, TreeNode treeNode) throws IOException { try { + ///////////////////////////////// + // construct the output object // + ///////////////////////////////// T output = outputClass.getConstructor().newInstance(); + ///////////////////////////////////////////////////////////////////////////////////////////////// + // set up a mapping between field names, and lambdas which will take a String (from the json), // + // and set it in the output object, doing type conversion as needed. // + // do this by iterating over methods on the output class that look like setters. // + ///////////////////////////////////////////////////////////////////////////////////////////////// Map> setterMap = new HashMap<>(); for(Method method : outputClass.getMethods()) { + ///////////////////////////////////////////////////////////// + // setters start with the word "set", and have 1 parameter // + ///////////////////////////////////////////////////////////// if(method.getName().startsWith("set") && method.getParameterTypes().length == 1) { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // get the parameter type, and the name of the field (remove set from the method name, and downshift the first letter) // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// Class parameterType = method.getParameterTypes()[0]; String fieldName = method.getName().substring(3, 4).toLowerCase(Locale.ROOT) + method.getName().substring(4); + /////////////////////////////////////////////////////////////////////////////////// + // put the entry in the map - where the value here is a consumer lambda function // + /////////////////////////////////////////////////////////////////////////////////// setterMap.put(fieldName, (String value) -> { try { + ////////////////////////////////////////////////////////////////////////////////////////////////// + // based on the parameter type, handle it differently - either type-converting (e.g., parseInt) // + // or gracefully ignoring, or failing. // + ////////////////////////////////////////////////////////////////////////////////////////////////// if(parameterType.equals(String.class)) { method.invoke(output, value); @@ -121,23 +155,20 @@ public class DeserializerUtils } else if(parameterType.equals(Class.class)) { - //////////////////////////////////////////////////////////// - // specifically do NOT try to handle Class type arguments // - //////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // specifically do NOT try to handle Class type arguments // + // we hit this when trying to de-serialize a QBackendMetaData, and we found its setBackendType(Class) method // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// } else if(parameterType.getPackageName().startsWith("java.")) { - //////////////////////////////////////////////////////////////////////// - // if we hit this, we might want to add an else-if to handle the type // - //////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we hit this, we might want to add an else-if to handle the type. // + // otherwise, either find some jackson annotation that makes sense, and apply it to the setter method, // + // or if no jackson annotation is right, then come up with annotation of our own. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// throw (new RuntimeException("Field " + fieldName + " is of an unhandled type " + parameterType.getName() + " when deserializing " + outputClass.getName())); } - else - { - //////////////////////////////////// - // gracefully ignore other types. // - //////////////////////////////////// - } } catch(IllegalAccessException | InvocationTargetException e) { @@ -153,9 +184,8 @@ public class DeserializerUtils } catch(Exception e) { - throw (new IOException("Error reflectively deserializing table details", e)); + throw (new IOException("Error deserializing json object into instance of " + outputClass.getName(), e)); } - } @@ -169,24 +199,35 @@ public class DeserializerUtils *******************************************************************************/ private static void deserializeBean(TreeNode treeNode, Map> setterMap) throws IOException { + /////////////////////////////////////////////////////// + // iterate over fields in the json object (treeNode) // + /////////////////////////////////////////////////////// Iterator fieldNamesIterator = treeNode.fieldNames(); while(fieldNamesIterator.hasNext()) { String fieldName = fieldNamesIterator.next(); + ////////////////////////////////////////////////////////////////////////// + // error if we find a field in the json that we don't have a setter for // + ////////////////////////////////////////////////////////////////////////// if(!setterMap.containsKey(fieldName)) { - throw (new IllegalArgumentException("Unexpected value: " + fieldName)); + throw (new IOException("Unexpected field (no corresponding setter): " + fieldName)); } + // call the setter - TreeNode fieldNode = treeNode.get(fieldName); if(fieldNode instanceof NullNode) { setterMap.get(fieldName).accept(null); } + else if(fieldNode instanceof TextNode textNode) + { + setterMap.get(fieldName).accept(textNode.asText()); + } else { - setterMap.get(fieldName).accept(((TextNode) fieldNode).asText()); + throw (new IOException("Unexpected node type (" + fieldNode.getClass() + ") for field: " + fieldName)); } } } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLCallback.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLCallback.java deleted file mode 100644 index e9f14ba7..00000000 --- a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLCallback.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; - - -import java.io.Serializable; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import com.kingsrook.qqq.backend.core.callbacks.QProcessCallback; -import com.kingsrook.qqq.backend.core.model.actions.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; - - -/******************************************************************************* - ** Provide callback functionality for the BasicETL process - *******************************************************************************/ -public class BasicETLCallback implements QProcessCallback -{ - - /******************************************************************************* - ** Get the filter query for this callback. - *******************************************************************************/ - @Override - public QQueryFilter getQueryFilter() - { - // todo - possibly get something from params? through state? added as a method arg? - return null; - } - - - - /******************************************************************************* - ** Get the field values for this callback. - *******************************************************************************/ - @SuppressWarnings("checkstyle:Indentation") - @Override - public Map getFieldValues(List fields) - { - Map rs = new HashMap<>(); - for(QFieldMetaData field : fields) - { - // TODO - replace this whole thing with our params mechanism - // TODO - add default methods to the interface that throw, presumably? - rs.put(field.getName(), switch(field.getName()) - { - case BasicETLProcess.FIELD_SOURCE_TABLE -> "personFile"; - case BasicETLProcess.FIELD_DESTINATION_TABLE -> "person"; - default -> throw new IllegalArgumentException("Unhandled field: " + field.getName()); - }); - } - return (rs); - } -} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java index c6dd0287..c06c764e 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java @@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; *******************************************************************************/ public class BasicETLProcess { + public static final String PROCESS_NAME = "etl.basic"; public static final String FIELD_SOURCE_TABLE = "sourceTable"; public static final String FIELD_DESTINATION_TABLE = "destinationTable"; public static final String FIELD_RECORD_COUNT = "recordCount"; @@ -70,7 +71,7 @@ public class BasicETLProcess .addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER))); return new QProcessMetaData() - .withName("etl.basic") + .withName(PROCESS_NAME) .addFunction(extractFunction) .addFunction(loadFunction); } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java b/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java index 3d995965..60fc22ed 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.state; import java.io.Serializable; import java.util.HashMap; import java.util.Map; +import java.util.Optional; /******************************************************************************* @@ -63,7 +64,7 @@ public class InMemoryStateProvider implements StateProviderInterface /******************************************************************************* - ** + ** Put a block of data, under a key, into the state store. *******************************************************************************/ @Override public void put(AbstractStateKey key, T data) @@ -74,14 +75,14 @@ public class InMemoryStateProvider implements StateProviderInterface /******************************************************************************* - ** + ** Get a block of data, under a key, from the state store. *******************************************************************************/ @Override - public T get(Class type, AbstractStateKey key) + public Optional get(Class type, AbstractStateKey key) { try { - return type.cast(map.get(key)); + return Optional.ofNullable(type.cast(map.get(key))); } catch(ClassCastException cce) { diff --git a/src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java b/src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java index c68daa5a..b2196746 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/state/StateProviderInterface.java @@ -23,10 +23,22 @@ package com.kingsrook.qqq.backend.core.state; import java.io.Serializable; +import java.util.Optional; /******************************************************************************* + ** QQQ state provider interface. Provides standard interface for various + ** implementations of how to store & retrieve user/process state data, like + ** sessions, or process data. Not like permanent record data - that is done in + ** Backend modules. ** + ** Different implementations may be: in-memory (non-persistent!!), or on-disk + ** (with the tradeoffs that has), in-database, in-cache-system, etc. + ** + ** Things which probably haven't been thought about here include: + ** - multi-layering. e.g., always have an in-memory layer on top of a more + ** persistent backend, but then how to avoid staleness in-memory? + * - cleanup. when do we ever purge things to avoid running out of memory/storage? *******************************************************************************/ public interface StateProviderInterface { @@ -39,5 +51,5 @@ public interface StateProviderInterface /******************************************************************************* ** Get a block of data, under a key, from the state store. *******************************************************************************/ - T get(Class type, AbstractStateKey key); + Optional get(Class type, AbstractStateKey key); } diff --git a/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java b/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java index 0c6e41d3..19ea144d 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/state/TempFileStateProvider.java @@ -26,6 +26,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.Serializable; +import java.util.Optional; import com.kingsrook.qqq.backend.core.utils.JsonUtils; import org.apache.commons.io.FileUtils; @@ -63,7 +64,7 @@ public class TempFileStateProvider implements StateProviderInterface /******************************************************************************* - ** + ** Put a block of data, under a key, into the state store. *******************************************************************************/ @Override public void put(AbstractStateKey key, T data) @@ -83,19 +84,19 @@ public class TempFileStateProvider implements StateProviderInterface /******************************************************************************* - ** + ** Get a block of data, under a key, from the state store. *******************************************************************************/ @Override - public T get(Class type, AbstractStateKey key) + public Optional get(Class type, AbstractStateKey key) { try { String json = FileUtils.readFileToString(new File("/tmp/" + key.toString())); - return JsonUtils.toObject(json, type); + return (Optional.of(JsonUtils.toObject(json, type))); } catch(FileNotFoundException fnfe) { - return (null); + return (Optional.empty()); } catch(IOException ie) { diff --git a/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java index cfb5fe9b..84eb7d83 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcessTest.java @@ -26,11 +26,6 @@ import com.kingsrook.qqq.backend.core.actions.RunProcessAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessRequest; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessResult; -import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -50,17 +45,12 @@ class BasicETLProcessTest @Test public void test() throws QException { - BasicETLProcess basicETLProcess = new BasicETLProcess(); - QProcessMetaData processMetaData = basicETLProcess.defineProcessMetaData(); - QInstance instance = TestUtils.defineInstance(); - RunProcessRequest request = new RunProcessRequest(instance); - - instance.addProcess(processMetaData); - defineFileBackendAndPersonFileTable(instance); - + RunProcessRequest request = new RunProcessRequest(TestUtils.defineInstance()); request.setSession(TestUtils.getMockSession()); - request.setProcessName(processMetaData.getName()); - request.setCallback(new BasicETLCallback()); // todo - uh, maybe a method on the process to get its callback? + request.setProcessName(BasicETLProcess.PROCESS_NAME); + request.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.defineTablePerson().getName()); + request.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.definePersonFileTable().getName()); + RunProcessResult result = new RunProcessAction().execute(request); assertNotNull(result); assertNull(result.getError()); @@ -68,27 +58,4 @@ class BasicETLProcessTest } - - /******************************************************************************* - ** Define the 'person' table used in standard tests. - *******************************************************************************/ - public static void defineFileBackendAndPersonFileTable(QInstance instance) - { - QTableMetaData personFileTable = new QTableMetaData() - .withName("personFile") - .withLabel("Person File") - .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) - .withPrimaryKeyField("id") - .withField(new QFieldMetaData("id", QFieldType.INTEGER)) - .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) - .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)) - .withField(new QFieldMetaData("firstName", QFieldType.STRING)) - .withField(new QFieldMetaData("lastName", QFieldType.STRING)) - .withField(new QFieldMetaData("birthDate", QFieldType.DATE)) - .withField(new QFieldMetaData("email", QFieldType.STRING)) - .withField(new QFieldMetaData("homeState", QFieldType.STRING).withPossibleValueSourceName("state")); - - instance.addTable(personFileTable); - } - } \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java b/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java index 193305db..8871e683 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java @@ -44,7 +44,7 @@ public class InMemoryStateProviderTest InMemoryStateProvider stateProvider = InMemoryStateProvider.getInstance(); UUIDStateKey key = new UUIDStateKey(); - Assertions.assertNull(stateProvider.get(QRecord.class, key), "Key not found in state should return null"); + Assertions.assertTrue(stateProvider.get(QRecord.class, key).isEmpty(), "Key not found in state should return empty"); } @@ -61,7 +61,7 @@ public class InMemoryStateProviderTest QRecord qRecord = new QRecord().withValue("uuid", uuid); stateProvider.put(key, qRecord); - QRecord qRecordFromState = stateProvider.get(QRecord.class, key); + QRecord qRecordFromState = stateProvider.get(QRecord.class, key).get(); Assertions.assertEquals(uuid, qRecordFromState.getValueString("uuid"), "Should read value from state persistence"); } diff --git a/src/test/java/com/kingsrook/qqq/backend/core/state/TempFileStateProviderTest.java b/src/test/java/com/kingsrook/qqq/backend/core/state/TempFileStateProviderTest.java index 4f361afd..6578d978 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/state/TempFileStateProviderTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/state/TempFileStateProviderTest.java @@ -44,7 +44,7 @@ public class TempFileStateProviderTest TempFileStateProvider stateProvider = TempFileStateProvider.getInstance(); UUIDStateKey key = new UUIDStateKey(); - Assertions.assertNull(stateProvider.get(QRecord.class, key), "Key not found in state should return null"); + Assertions.assertTrue(stateProvider.get(QRecord.class, key).isEmpty(), "Key not found in state should return empty"); } @@ -61,7 +61,7 @@ public class TempFileStateProviderTest QRecord qRecord = new QRecord().withValue("uuid", uuid); stateProvider.put(key, qRecord); - QRecord qRecordFromState = stateProvider.get(QRecord.class, key); + QRecord qRecordFromState = stateProvider.get(QRecord.class, key).get(); Assertions.assertEquals(uuid, qRecordFromState.getValueString("uuid"), "Should read value from state persistence"); } diff --git a/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index d15e4b65..92862ea8 100644 --- a/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.utils; import java.util.List; +import com.kingsrook.qqq.backend.core.interfaces.mock.MockFunctionBody; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference; @@ -43,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaDa import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListView; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.mock.MockAuthenticationModule; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; /******************************************************************************* @@ -53,6 +55,8 @@ public class TestUtils { public static String DEFAULT_BACKEND_NAME = "default"; + + /******************************************************************************* ** Define the instance used in standard tests. ** @@ -63,9 +67,11 @@ public class TestUtils qInstance.setAuthentication(defineAuthentication()); qInstance.addBackend(defineBackend()); qInstance.addTable(defineTablePerson()); + qInstance.addTable(definePersonFileTable()); qInstance.addPossibleValueSource(defineStatesPossibleValueSource()); qInstance.addProcess(defineProcessGreetPeople()); qInstance.addProcess(defineProcessAddToPeoplesAge()); + qInstance.addProcess(new BasicETLProcess().defineProcessMetaData()); return (qInstance); } @@ -118,7 +124,7 @@ public class TestUtils return new QTableMetaData() .withName("person") .withLabel("Person") - .withBackendName(defineBackend().getName()) + .withBackendName(DEFAULT_BACKEND_NAME) .withPrimaryKeyField("id") .withField(new QFieldMetaData("id", QFieldType.INTEGER)) .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) @@ -132,6 +138,21 @@ public class TestUtils + /******************************************************************************* + ** Define a 2nd version of the 'person' table for this test (pretend it's backed by a file) + *******************************************************************************/ + public static QTableMetaData definePersonFileTable() + { + return (new QTableMetaData() + .withName("personFile") + .withLabel("Person File") + .withBackendName(DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") + .withFields(TestUtils.defineTablePerson().getFields())); + } + + + /******************************************************************************* ** Define the 'greet people' process *******************************************************************************/ @@ -143,7 +164,7 @@ public class TestUtils .addFunction(new QFunctionMetaData() .withName("prepare") .withCode(new QCodeReference() - .withName("com.kingsrook.qqq.backend.core.interfaces.mock.MockFunctionBody") + .withName(MockFunctionBody.class.getName()) .withCodeType(QCodeType.JAVA) .withCodeUsage(QCodeUsage.FUNCTION)) // todo - needed, or implied in this context? .withInputData(new QFunctionInputMetaData()