From a86f42f373d660c6c5271286f06d20eb0af6ece2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 22 Aug 2022 08:34:55 -0500 Subject: [PATCH 01/28] QQQ-37 initial buidout of StreamedETLWithFrontendProcess --- .../core/actions/async/AsyncJobCallback.java | 14 +- .../core/actions/async/AsyncJobManager.java | 11 ++ .../core/actions/async/AsyncJobStatus.java | 24 +++ .../actions/async/AsyncRecordPipeLoop.java | 157 ++++++++++++++++++ .../core/actions/customizers/QCodeLoader.java | 41 +++++ .../actions/interfaces/InsertInterface.java | 11 +- .../actions/interfaces/QActionInterface.java | 23 +++ .../core/actions/reporting/RecordPipe.java | 31 +++- .../actions/tables/query/QueryOutput.java | 4 +- .../actions/tables/update/UpdateInput.java | 38 ++++- .../metadata/processes/QProcessMetaData.java | 10 ++ .../AbstractExtractFunction.java | 58 +++++++ .../AbstractLoadFunction.java | 85 ++++++++++ .../AbstractTransformFunction.java | 61 +++++++ .../BaseStreamedETLStep.java | 49 ++++++ .../StreamedETLExecuteStep.java | 146 ++++++++++++++++ .../StreamedETLPreviewStep.java | 95 +++++++++++ .../StreamedETLWithFrontendProcess.java | 89 ++++++++++ .../rdbms/actions/AbstractRDBMSAction.java | 32 +++- .../rdbms/actions/RDBMSInsertAction.java | 23 --- .../rdbms/actions/RDBMSQueryAction.java | 6 + .../rdbms/actions/RDBMSUpdateAction.java | 39 ++++- 22 files changed, 998 insertions(+), 49 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QActionInterface.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractFunction.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadFunction.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformFunction.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobCallback.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobCallback.java index fd5993c3..ec9ff1de 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobCallback.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobCallback.java @@ -31,7 +31,6 @@ import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; ** Argument passed to an AsyncJob when it runs, which can be used to communicate ** data back out of the job. ** - ** TODO - future - allow cancellation to be indicated here? *******************************************************************************/ public class AsyncJobCallback { @@ -107,4 +106,17 @@ public class AsyncJobCallback AsyncJobManager.getStateProvider().put(new UUIDAndTypeStateKey(jobUUID, StateType.ASYNC_JOB_STATUS), asyncJobStatus); } + + + /******************************************************************************* + ** Check if the asyncJobStatus had a cancellation requested. + ** + ** TODO - concern about multiple threads writing this object to a non-in-memory + ** state provider, and this value getting lost... + *******************************************************************************/ + public boolean wasCancelRequested() + { + return (this.asyncJobStatus.getCancelRequested()); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java index 63766320..fbf95f07 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java @@ -183,4 +183,15 @@ public class AsyncJobManager // return TempFileStateProvider.getInstance(); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public void cancelJob(String jobUUID) + { + Optional jobStatus = getJobStatus(jobUUID); + jobStatus.ifPresent(asyncJobStatus -> asyncJobStatus.setCancelRequested(true)); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java index 83810512..3ec54516 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java @@ -37,6 +37,8 @@ public class AsyncJobStatus implements Serializable private Integer total; private Exception caughtException; + private boolean cancelRequested; + /******************************************************************************* @@ -163,4 +165,26 @@ public class AsyncJobStatus implements Serializable { this.caughtException = caughtException; } + + + + /******************************************************************************* + ** Getter for cancelRequested + ** + *******************************************************************************/ + public boolean getCancelRequested() + { + return cancelRequested; + } + + + + /******************************************************************************* + ** Setter for cancelRequested + ** + *******************************************************************************/ + public void setCancelRequested(boolean cancelRequested) + { + this.cancelRequested = cancelRequested; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java new file mode 100644 index 00000000..554ec0a1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java @@ -0,0 +1,157 @@ +package com.kingsrook.qqq.backend.core.actions.async; + + +import java.io.Serializable; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class AsyncRecordPipeLoop +{ + private static final Logger LOG = LogManager.getLogger(AsyncRecordPipeLoop.class); + + private static final int TIMEOUT_AFTER_NO_RECORDS_MS = 10 * 60 * 1000; + + private static final int MAX_SLEEP_MS = 1000; + private static final int INIT_SLEEP_MS = 10; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public int run(String jobName, Integer recordLimit, RecordPipe recordPipe, UnsafeFunction job, UnsafeSupplier consumer) throws QException + { + /////////////////////////////////////////////////// + // start the extraction function as an async job // + /////////////////////////////////////////////////// + AsyncJobManager asyncJobManager = new AsyncJobManager(); + String jobUUID = asyncJobManager.startJob(jobName, job::apply); + LOG.info("Started job [" + jobUUID + "] for record pipe streaming"); + + AsyncJobState jobState = AsyncJobState.RUNNING; + AsyncJobStatus asyncJobStatus = null; + + int recordCount = 0; + int nextSleepMillis = INIT_SLEEP_MS; + long lastReceivedRecordsAt = System.currentTimeMillis(); + long jobStartTime = System.currentTimeMillis(); + + while(jobState.equals(AsyncJobState.RUNNING)) + { + if(recordPipe.countAvailableRecords() == 0) + { + /////////////////////////////////////////////////////////// + // if the pipe is empty, sleep to let the producer work. // + // todo - smarter sleep? like get notified vs. sleep? // + /////////////////////////////////////////////////////////// + LOG.info("No records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work"); + SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS); + nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS); + + long timeSinceLastReceivedRecord = System.currentTimeMillis() - lastReceivedRecordsAt; + if(timeSinceLastReceivedRecord > TIMEOUT_AFTER_NO_RECORDS_MS) + { + throw (new QException("Job appears to have stopped producing records (last record received " + timeSinceLastReceivedRecord + " ms ago).")); + } + } + else + { + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the pipe has records, consume them. reset the sleep timer so if we sleep again it'll be short. // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + lastReceivedRecordsAt = System.currentTimeMillis(); + nextSleepMillis = INIT_SLEEP_MS; + + recordCount += consumer.get(); + LOG.info(String.format("Processed %,d records so far", recordCount)); + + if(recordLimit != null && recordCount >= recordLimit) + { + asyncJobManager.cancelJob(jobUUID); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // in case the extract function doesn't recognize the cancellation request, // + // tell the pipe to "terminate" - meaning - flush its queue and just noop when given new records. // + // this should prevent anyone writing to such a pipe from potentially filling & blocking. // + //////////////////////////////////////////////////////////////////////////////////////////////////// + recordPipe.terminate(); + + break; + } + } + + ////////////////////////////// + // refresh the job's status // + ////////////////////////////// + Optional optionalAsyncJobStatus = asyncJobManager.getJobStatus(jobUUID); + if(optionalAsyncJobStatus.isEmpty()) + { + ///////////////////////////////////////////////// + // todo - ... maybe some version of try-again? // + ///////////////////////////////////////////////// + throw (new QException("Could not get status of job [" + jobUUID + "]")); + } + asyncJobStatus = optionalAsyncJobStatus.get(); + jobState = asyncJobStatus.getState(); + } + + LOG.info("Job [" + jobUUID + "] completed with status: " + asyncJobStatus); + + /////////////////////////////////// + // propagate errors from the job // + /////////////////////////////////// + if(asyncJobStatus != null && asyncJobStatus.getState().equals(AsyncJobState.ERROR)) + { + throw (new QException("Job failed with an error", asyncJobStatus.getCaughtException())); + } + + ////////////////////////////////////////////////////// + // send the final records to transform & load steps // + ////////////////////////////////////////////////////// + recordCount += consumer.get(); + + long endTime = System.currentTimeMillis(); + LOG.info(String.format("Processed %,d records", recordCount) + + String.format(" at end of job in %,d ms (%.2f records/second).", (endTime - jobStartTime), 1000d * (recordCount / (.001d + (endTime - jobStartTime))))); + + return (recordCount); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @FunctionalInterface + public interface UnsafeFunction + { + /******************************************************************************* + ** + *******************************************************************************/ + R apply(T t) throws QException; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @FunctionalInterface + public interface UnsafeSupplier + { + /******************************************************************************* + ** + *******************************************************************************/ + T get() throws QException; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java index 8e7c0dfc..bcc8a7a7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.customizers; import java.util.Optional; import java.util.function.Function; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -93,4 +94,44 @@ public class QCodeLoader } } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static T getBackendStep(Class expectedType, QCodeReference codeReference) + { + if(codeReference == null) + { + return (null); + } + + if(!codeReference.getCodeType().equals(QCodeType.JAVA)) + { + /////////////////////////////////////////////////////////////////////////////////////// + // todo - 1) support more languages, 2) wrap them w/ java Functions here, 3) profit! // + /////////////////////////////////////////////////////////////////////////////////////// + throw (new IllegalArgumentException("Only JAVA BackendSteps are supported at this time.")); + } + + try + { + Class customizerClass = Class.forName(codeReference.getName()); + return ((T) customizerClass.getConstructor().newInstance()); + } + catch(Exception e) + { + LOG.error("Error initializing customizer: " + codeReference); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // return null here - under the assumption that during normal run-time operations, we'll never hit here // + // as we'll want to validate all functions in the instance validator at startup time (and IT will throw // + // if it finds an invalid code reference // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + return (null); + } + } + + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java index dc90de66..3eb2738d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java @@ -22,7 +22,6 @@ package com.kingsrook.qqq.backend.core.actions.interfaces; -import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; @@ -32,19 +31,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; ** Interface for the Insert action. ** *******************************************************************************/ -public interface InsertInterface +public interface InsertInterface extends QActionInterface { /******************************************************************************* ** *******************************************************************************/ InsertOutput execute(InsertInput insertInput) throws QException; - /******************************************************************************* - ** - *******************************************************************************/ - default QBackendTransaction openTransaction(InsertInput insertInput) throws QException - { - return (new QBackendTransaction()); - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QActionInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QActionInterface.java new file mode 100644 index 00000000..cac3f87f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QActionInterface.java @@ -0,0 +1,23 @@ +package com.kingsrook.qqq.backend.core.actions.interfaces; + + +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface QActionInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + default QBackendTransaction openTransaction(AbstractTableActionInput input) throws QException + { + return (new QBackendTransaction()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java index 3703a841..e3eec254 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java @@ -42,16 +42,36 @@ public class RecordPipe private ArrayBlockingQueue queue = new ArrayBlockingQueue<>(1_000); + private boolean isTerminated = false; + + + /******************************************************************************* + ** Turn off the pipe. Stop accepting new records (just ignore them in the add + ** method). Clear the existing queue. Don't return any more records. Note that + ** if consumeAvailableRecords was running in another thread, it may still return + ** some records that it read before this call. + *******************************************************************************/ + public void terminate() + { + isTerminated = true; + queue.clear(); + } + + /******************************************************************************* ** Add a record to the pipe - ** Returns true iff the record fit in the pipe; false if the pipe is currently full. *******************************************************************************/ public void addRecord(QRecord record) { + if(isTerminated) + { + return; + } + boolean offerResult = queue.offer(record); - while(!offerResult) + while(!offerResult && !isTerminated) { LOG.debug("Record pipe.add failed (due to full pipe). Blocking."); SleepUtils.sleep(100, TimeUnit.MILLISECONDS); @@ -78,7 +98,7 @@ public class RecordPipe { List rs = new ArrayList<>(); - while(true) + while(!isTerminated) { QRecord record = queue.poll(); if(record == null) @@ -98,6 +118,11 @@ public class RecordPipe *******************************************************************************/ public int countAvailableRecords() { + if(isTerminated) + { + return (0); + } + return (queue.size()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java index 96340a69..9412e0d0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java @@ -25,8 +25,8 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.io.Serializable; import java.util.List; import java.util.function.Function; -import com.kingsrook.qqq.backend.core.actions.customizers.CustomizerLoader; import com.kingsrook.qqq.backend.core.actions.customizers.Customizers; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import org.apache.logging.log4j.LogManager; @@ -62,7 +62,7 @@ public class QueryOutput extends AbstractActionOutput implements Serializable storage = new QueryOutputList(); } - postQueryRecordCustomizer = (Function) CustomizerLoader.getTableCustomizerFunction(queryInput.getTable(), Customizers.POST_QUERY_RECORD); + postQueryRecordCustomizer = (Function) QCodeLoader.getTableCustomizerFunction(queryInput.getTable(), Customizers.POST_QUERY_RECORD); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java index fca9ee57..adabb9f4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.update; import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -34,7 +35,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; *******************************************************************************/ public class UpdateInput extends AbstractTableActionInput { - private List records; + private QBackendTransaction transaction; + private List records; //////////////////////////////////////////////////////////////////////////////////////////// // allow a caller to specify that they KNOW this optimization (e.g., in SQL) can be made. // @@ -65,6 +67,40 @@ public class UpdateInput extends AbstractTableActionInput + /******************************************************************************* + ** Getter for transaction + ** + *******************************************************************************/ + public QBackendTransaction getTransaction() + { + return transaction; + } + + + + /******************************************************************************* + ** Setter for transaction + ** + *******************************************************************************/ + public void setTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + } + + + + /******************************************************************************* + ** Fluent setter for transaction + ** + *******************************************************************************/ + public UpdateInput withTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + return (this); + } + + + /******************************************************************************* ** Getter for records ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index 07105a34..a9fa5611 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -228,6 +228,16 @@ public class QProcessMetaData implements QAppChildMetaData + /******************************************************************************* + ** Wrapper to getStep, that internally casts to FrontendStepMetaData + *******************************************************************************/ + public QFrontendStepMetaData getFrontendStep(String name) + { + return (QFrontendStepMetaData) getStep(name); + } + + + /******************************************************************************* ** Get a list of all of the input fields used by all the steps in this process. *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractFunction.java new file mode 100644 index 00000000..fbe41445 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractFunction.java @@ -0,0 +1,58 @@ +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class AbstractExtractFunction implements BackendStep +{ + private RecordPipe recordPipe; + private Integer limit; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void setRecordPipe(RecordPipe recordPipe) + { + this.recordPipe = recordPipe; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public RecordPipe getRecordPipe() + { + return recordPipe; + } + + + + /******************************************************************************* + ** Getter for limit + ** + *******************************************************************************/ + public Integer getLimit() + { + return limit; + } + + + + /******************************************************************************* + ** Setter for limit + ** + *******************************************************************************/ + public void setLimit(Integer limit) + { + this.limit = limit; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadFunction.java new file mode 100644 index 00000000..500ed3f6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadFunction.java @@ -0,0 +1,85 @@ +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class AbstractLoadFunction implements BackendStep +{ + private List inputRecordPage = new ArrayList<>(); + private List outputRecordPage = new ArrayList<>(); + + protected QBackendTransaction transaction; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QBackendTransaction openTransaction(RunBackendStepInput runBackendStepInput) throws QException + { + this.transaction = doOpenTransaction(runBackendStepInput); + return (transaction); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected abstract QBackendTransaction doOpenTransaction(RunBackendStepInput runBackendStepInput) throws QException; + + + + /******************************************************************************* + ** Getter for recordPage + ** + *******************************************************************************/ + public List getInputRecordPage() + { + return inputRecordPage; + } + + + + /******************************************************************************* + ** Setter for recordPage + ** + *******************************************************************************/ + public void setInputRecordPage(List inputRecordPage) + { + this.inputRecordPage = inputRecordPage; + } + + + + /******************************************************************************* + ** Getter for outputRecordPage + ** + *******************************************************************************/ + public List getOutputRecordPage() + { + return outputRecordPage; + } + + + + /******************************************************************************* + ** Setter for outputRecordPage + ** + *******************************************************************************/ + public void setOutputRecordPage(List outputRecordPage) + { + this.outputRecordPage = outputRecordPage; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformFunction.java new file mode 100644 index 00000000..c21f4c89 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformFunction.java @@ -0,0 +1,61 @@ +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class AbstractTransformFunction implements BackendStep +{ + private List inputRecordPage = new ArrayList<>(); + private List outputRecordPage = new ArrayList<>(); + + + + /******************************************************************************* + ** Getter for recordPage + ** + *******************************************************************************/ + public List getInputRecordPage() + { + return inputRecordPage; + } + + + + /******************************************************************************* + ** Setter for recordPage + ** + *******************************************************************************/ + public void setInputRecordPage(List inputRecordPage) + { + this.inputRecordPage = inputRecordPage; + } + + + + /******************************************************************************* + ** Getter for outputRecordPage + ** + *******************************************************************************/ + public List getOutputRecordPage() + { + return outputRecordPage; + } + + + + /******************************************************************************* + ** Setter for outputRecordPage + ** + *******************************************************************************/ + public void setOutputRecordPage(List outputRecordPage) + { + this.outputRecordPage = outputRecordPage; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java new file mode 100644 index 00000000..cd239a5f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java @@ -0,0 +1,49 @@ +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; + + +/******************************************************************************* + ** Base class for the StreamedETL preview & execute steps + *******************************************************************************/ +public class BaseStreamedETLStep +{ + protected static final int IN_MEMORY_RECORD_LIMIT = 20; + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected AbstractExtractFunction getExtractFunction(RunBackendStepInput runBackendStepInput) + { + QCodeReference codeReference = (QCodeReference) runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_EXTRACT_CODE); + return (QCodeLoader.getBackendStep(AbstractExtractFunction.class, codeReference)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected AbstractTransformFunction getTransformFunction(RunBackendStepInput runBackendStepInput) + { + QCodeReference codeReference = (QCodeReference) runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_TRANSFORM_CODE); + return (QCodeLoader.getBackendStep(AbstractTransformFunction.class, codeReference)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected AbstractLoadFunction getLoadFunction(RunBackendStepInput runBackendStepInput) + { + QCodeReference codeReference = (QCodeReference) runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_LOAD_CODE); + return (QCodeLoader.getBackendStep(AbstractLoadFunction.class, codeReference)); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java new file mode 100644 index 00000000..f484ace5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java @@ -0,0 +1,146 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; + + +/******************************************************************************* + ** Backend step to do a execute a streamed ETL job + *******************************************************************************/ +public class StreamedETLExecuteStep extends BaseStreamedETLStep implements BackendStep +{ + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + @SuppressWarnings("checkstyle:indentation") + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + QBackendTransaction transaction = null; + + try + { + /////////////////////////////////////////////////////// + // set up the extract, transform, and load functions // + /////////////////////////////////////////////////////// + RecordPipe recordPipe = new RecordPipe(); + AbstractExtractFunction extractFunction = getExtractFunction(runBackendStepInput); + extractFunction.setRecordPipe(recordPipe); + + AbstractTransformFunction transformFunction = getTransformFunction(runBackendStepInput); + AbstractLoadFunction loadFunction = getLoadFunction(runBackendStepInput); + + transaction = loadFunction.openTransaction(runBackendStepInput); + + List loadedRecordList = new ArrayList<>(); + int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Execute>ExtractFunction", null, recordPipe, (status) -> + { + extractFunction.run(runBackendStepInput, runBackendStepOutput); + return (runBackendStepOutput); + }, + () -> (consumeRecordsFromPipe(recordPipe, transformFunction, loadFunction, runBackendStepInput, runBackendStepOutput, loadedRecordList)) + ); + + runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount); + runBackendStepOutput.setRecords(loadedRecordList); + + ///////////////////// + // commit the work // + ///////////////////// + transaction.commit(); + } + catch(Exception e) + { + //////////////////////////////////////////////////////////////////////////////// + // rollback the work, then re-throw the error for up-stream to catch & report // + //////////////////////////////////////////////////////////////////////////////// + if(transaction != null) + { + transaction.rollback(); + } + throw (e); + } + finally + { + //////////////////////////////////////////////////////////// + // always close our transactions (e.g., jdbc connections) // + //////////////////////////////////////////////////////////// + if(transaction != null) + { + transaction.close(); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformFunction transformFunction, AbstractLoadFunction loadFunction, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List loadedRecordList) throws QException + { + /////////////////////////////////// + // get the records from the pipe // + /////////////////////////////////// + List qRecords = recordPipe.consumeAvailableRecords(); + + ///////////////////////////////////////////////////// + // pass the records through the transform function // + ///////////////////////////////////////////////////// + transformFunction.setInputRecordPage(qRecords); + transformFunction.setOutputRecordPage(new ArrayList<>()); + transformFunction.run(runBackendStepInput, runBackendStepOutput); + + //////////////////////////////////////////////// + // pass the records through the load function // + //////////////////////////////////////////////// + loadFunction.setInputRecordPage(transformFunction.getOutputRecordPage()); + loadFunction.setOutputRecordPage(new ArrayList<>()); + loadFunction.run(runBackendStepInput, runBackendStepOutput); + + /////////////////////////////////////////////////////// + // copy a small number of records to the output list // + /////////////////////////////////////////////////////// + int i = 0; + while(loadedRecordList.size() < IN_MEMORY_RECORD_LIMIT && i < loadFunction.getOutputRecordPage().size()) + { + loadedRecordList.add(loadFunction.getOutputRecordPage().get(i++)); + } + + return (qRecords.size()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java new file mode 100644 index 00000000..f9cc4143 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -0,0 +1,95 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Backend step to do a preview of a full streamed ETL job + *******************************************************************************/ +public class StreamedETLPreviewStep extends BaseStreamedETLStep implements BackendStep +{ + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + @SuppressWarnings("checkstyle:indentation") + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + RecordPipe recordPipe = new RecordPipe(); + AbstractExtractFunction extractFunction = getExtractFunction(runBackendStepInput); + extractFunction.setLimit(IN_MEMORY_RECORD_LIMIT); // todo - process field? + extractFunction.setRecordPipe(recordPipe); + + AbstractTransformFunction transformFunction = getTransformFunction(runBackendStepInput); + + List transformedRecordList = new ArrayList<>(); + new AsyncRecordPipeLoop().run("StreamedETL>Preview>ExtractFunction", IN_MEMORY_RECORD_LIMIT, recordPipe, (status) -> + { + extractFunction.run(runBackendStepInput, runBackendStepOutput); + return (runBackendStepOutput); + }, + () -> (consumeRecordsFromPipe(recordPipe, transformFunction, runBackendStepInput, runBackendStepOutput, transformedRecordList)) + ); + + runBackendStepOutput.setRecords(transformedRecordList); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformFunction transformFunction, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List transformedRecordList) throws QException + { + /////////////////////////////////// + // get the records from the pipe // + /////////////////////////////////// + List qRecords = recordPipe.consumeAvailableRecords(); + + ///////////////////////////////////////////////////// + // pass the records through the transform function // + ///////////////////////////////////////////////////// + transformFunction.setInputRecordPage(qRecords); + transformFunction.run(runBackendStepInput, runBackendStepOutput); + + //////////////////////////////////////////////////// + // add the transformed records to the output list // + //////////////////////////////////////////////////// + transformedRecordList.addAll(transformFunction.getOutputRecordPage()); + + return (qRecords.size()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java new file mode 100644 index 00000000..4b4ba456 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -0,0 +1,89 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; + + +/******************************************************************************* + ** Definition for Streamed ETL process that includes a frontend. + ** + *******************************************************************************/ +public class StreamedETLWithFrontendProcess +{ + public static final String PROCESS_NAME = "etl.streamedWithFrontend"; + + public static final String STEP_NAME_PREVIEW = "preview"; + public static final String STEP_NAME_REVIEW = "review"; + public static final String STEP_NAME_EXECUTE = "execute"; + public static final String STEP_NAME_RESULT = "result"; + + public static final String FIELD_EXTRACT_CODE = "extract"; + public static final String FIELD_TRANSFORM_CODE = "transform"; + public static final String FIELD_LOAD_CODE = "load"; + + public static final String FIELD_SOURCE_TABLE = "sourceTable"; + public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + public static final String FIELD_MAPPING_JSON = "mappingJSON"; + public static final String FIELD_RECORD_COUNT = "recordCount"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QProcessMetaData defineProcessMetaData() + { + QStepMetaData previewStep = new QBackendStepMetaData() + .withName(STEP_NAME_PREVIEW) + .withCode(new QCodeReference(StreamedETLPreviewStep.class)) + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData().withName(FIELD_EXTRACT_CODE)) + .withField(new QFieldMetaData().withName(FIELD_TRANSFORM_CODE))); + + QFrontendStepMetaData reviewStep = new QFrontendStepMetaData() + .withName(STEP_NAME_REVIEW); + + QStepMetaData executeStep = new QBackendStepMetaData() + .withName(STEP_NAME_EXECUTE) + .withCode(new QCodeReference(StreamedETLExecuteStep.class)) + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData().withName(FIELD_LOAD_CODE))); + + QFrontendStepMetaData resultStep = new QFrontendStepMetaData() + .withName(STEP_NAME_RESULT); + + return new QProcessMetaData() + .withName(PROCESS_NAME) + .addStep(previewStep) + .addStep(reviewStep) + .addStep(executeStep) + .addStep(resultStep); + } +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index feafe6c9..3ec0545f 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -29,6 +29,9 @@ import java.util.ArrayList; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.interfaces.QActionInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -40,13 +43,18 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* ** Base class for all core actions in the RDBMS module. *******************************************************************************/ -public abstract class AbstractRDBMSAction +public abstract class AbstractRDBMSAction implements QActionInterface { + private static final Logger LOG = LogManager.getLogger(AbstractRDBMSAction.class); + + /******************************************************************************* ** Get the table name to use in the RDBMS from a QTableMetaData. @@ -319,4 +327,26 @@ public abstract class AbstractRDBMSAction { return fieldType == QFieldType.STRING || fieldType == QFieldType.TEXT || fieldType == QFieldType.HTML || fieldType == QFieldType.PASSWORD; } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QBackendTransaction openTransaction(AbstractTableActionInput input) throws QException + { + try + { + LOG.info("Opening transaction"); + Connection connection = getConnection(input); + + return (new RDBMSTransaction(connection)); + } + catch(Exception e) + { + throw new QException("Error opening transaction: " + e.getMessage(), e); + } + } + } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index 7bdd715b..f4a08148 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -28,7 +28,6 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; @@ -160,26 +159,4 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte } } - - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public QBackendTransaction openTransaction(InsertInput insertInput) throws QException - { - try - { - LOG.info("Opening transaction"); - Connection connection = getConnection(insertInput); - - return (new RDBMSTransaction(connection)); - } - catch(Exception e) - { - throw new QException("Error opening transaction: " + e.getMessage(), e); - } - } - - } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 566a220e..fa08c192 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -123,6 +123,12 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf } queryOutput.addRecord(record); + + if(queryInput.getAsyncJobCallback().wasCancelRequested()) + { + LOG.info("Breaking query job, as requested."); + break; + } } }), params); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java index d7ef8739..559db22c 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java @@ -114,14 +114,37 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte outputRecords.add(outputRecord); } - try(Connection connection = getConnection(updateInput)) + try { - ///////////////////////////////////////////////////////////////////////////////////////////// - // process each distinct list of fields being updated (e.g., each different SQL statement) // - ///////////////////////////////////////////////////////////////////////////////////////////// - for(List fieldsBeingUpdated : recordsByFieldBeingUpdated.keySet()) + Connection connection; + boolean needToCloseConnection = false; + if(updateInput.getTransaction() != null && updateInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction) { - updateRecordsWithMatchingListOfFields(updateInput, connection, table, recordsByFieldBeingUpdated.get(fieldsBeingUpdated), fieldsBeingUpdated); + LOG.debug("Using connection from insertInput [" + rdbmsTransaction.getConnection() + "]"); + connection = rdbmsTransaction.getConnection(); + } + else + { + connection = getConnection(updateInput); + needToCloseConnection = true; + } + + try + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // process each distinct list of fields being updated (e.g., each different SQL statement) // + ///////////////////////////////////////////////////////////////////////////////////////////// + for(List fieldsBeingUpdated : recordsByFieldBeingUpdated.keySet()) + { + updateRecordsWithMatchingListOfFields(updateInput, connection, table, recordsByFieldBeingUpdated.get(fieldsBeingUpdated), fieldsBeingUpdated); + } + } + finally + { + if(needToCloseConnection) + { + connection.close(); + } } return rs; @@ -191,7 +214,6 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte - /******************************************************************************* ** *******************************************************************************/ @@ -276,6 +298,8 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte return (true); } + + /******************************************************************************* ** *******************************************************************************/ @@ -285,5 +309,4 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte updateInput.getAsyncJobCallback().updateStatus(statusCounter, updateInput.getRecords().size()); } - } From 21d66cc7fcf3d919e496b4e90c4ee65a3c2fdb69 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 22 Aug 2022 10:07:55 -0500 Subject: [PATCH 02/28] QQQ-37 added test coverage for StreamedETLWithFrontendProcess --- .../model/metadata/code/QCodeReference.java | 3 +- .../StreamedETLExecuteStep.java | 5 +- .../StreamedETLWithFrontendProcessTest.java | 179 ++++++++++++++++++ 3 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeReference.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeReference.java index 5a82f004..40eff301 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeReference.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeReference.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.code; +import java.io.Serializable; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; @@ -30,7 +31,7 @@ import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvide ** Pointer to code to be ran by the qqq framework, e.g., for custom behavior - ** maybe process steps, maybe customization to a table, etc. *******************************************************************************/ -public class QCodeReference +public class QCodeReference implements Serializable { private String name; private QCodeType codeType; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java index f484ace5..2a00a5cc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java @@ -80,7 +80,10 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe ///////////////////// // commit the work // ///////////////////// - transaction.commit(); + if(transaction != null) + { + transaction.commit(); + } } catch(Exception e) { diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java new file mode 100644 index 00000000..73d8792e --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java @@ -0,0 +1,179 @@ +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for StreamedETLWithFrontendProcess + *******************************************************************************/ +class StreamedETLWithFrontendProcessTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData(); + process.setTableName(TestUtils.TABLE_NAME_SHAPE); + + for(QFieldMetaData inputField : process.getInputFields()) + { + if(StreamedETLWithFrontendProcess.FIELD_EXTRACT_CODE.equals(inputField.getName())) + { + inputField.setDefaultValue(new QCodeReference(TestExtractStep.class)); + } + else if(StreamedETLWithFrontendProcess.FIELD_TRANSFORM_CODE.equals(inputField.getName())) + { + inputField.setDefaultValue(new QCodeReference(TestTransformStep.class)); + } + else if(StreamedETLWithFrontendProcess.FIELD_LOAD_CODE.equals(inputField.getName())) + { + inputField.setDefaultValue(new QCodeReference(TestLoadStep.class)); + } + } + + QInstance instance = TestUtils.defineInstance(); + instance.addProcess(process); + + InsertInput insertInput = new InsertInput(instance); + insertInput.setSession(TestUtils.getMockSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_SHAPE); + insertInput.setRecords(List.of( + new QRecord().withTableName(TestUtils.TABLE_NAME_SHAPE).withValue("id", 1).withValue("name", "Circle"), + new QRecord().withTableName(TestUtils.TABLE_NAME_SHAPE).withValue("id", 2).withValue("name", "Triangle"), + new QRecord().withTableName(TestUtils.TABLE_NAME_SHAPE).withValue("id", 3).withValue("name", "Square") + )); + new InsertAction().execute(insertInput); + + List preList = TestUtils.queryTable(TestUtils.TABLE_NAME_SHAPE); + + RunProcessInput request = new RunProcessInput(instance); + request.setSession(TestUtils.getMockSession()); + request.setProcessName(process.getName()); + request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + + RunProcessOutput result = new RunProcessAction().execute(request); + assertNotNull(result); + + assertTrue(result.getException().isEmpty()); + + List postList = TestUtils.queryTable(TestUtils.TABLE_NAME_SHAPE); + assertEquals(6, postList.size()); + assertThat(postList) + .anyMatch(qr -> qr.getValue("name").equals("Circle")) + .anyMatch(qr -> qr.getValue("name").equals("Triangle")) + .anyMatch(qr -> qr.getValue("name").equals("Square")) + .anyMatch(qr -> qr.getValue("name").equals("Transformed: Circle")) + .anyMatch(qr -> qr.getValue("name").equals("Transformed: Triangle")) + .anyMatch(qr -> qr.getValue("name").equals("Transformed: Square")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestExtractStep extends AbstractExtractFunction + { + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance()); + queryInput.setSession(runBackendStepInput.getSession()); + queryInput.setTableName(TestUtils.TABLE_NAME_SHAPE); + queryInput.setRecordPipe(getRecordPipe()); + new QueryAction().execute(queryInput); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestTransformStep extends AbstractTransformFunction + { + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + for(QRecord qRecord : getInputRecordPage()) + { + QRecord newQRecord = new QRecord(); + newQRecord.setValue("id", null); + newQRecord.setValue("name", "Transformed: " + qRecord.getValueString("name")); + getOutputRecordPage().add(newQRecord); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestLoadStep extends AbstractLoadFunction + { + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance()); + insertInput.setSession(runBackendStepInput.getSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_SHAPE); + insertInput.setTransaction(transaction); + insertInput.setRecords(getInputRecordPage()); + new InsertAction().execute(insertInput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected QBackendTransaction doOpenTransaction(RunBackendStepInput runBackendStepInput) throws QException + { + return null; + } + } + +} \ No newline at end of file From 937304e7f18be32d255e5035da4a178c685fe9de Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 22 Aug 2022 10:38:02 -0500 Subject: [PATCH 03/28] QQQ-38 Initial build of app home page widgets --- .../core/actions/customizers/QCodeLoader.java | 38 +++ .../dashboard/AbstractWidgetRenderer.java | 20 ++ .../actions/dashboard/WidgetDataLoader.java | 26 ++ .../actions/tables/query/QueryOutput.java | 4 +- .../model/dashboard/widgets/BarChart.java | 299 ++++++++++++++++++ .../core/model/metadata/QInstance.java | 59 ++++ .../metadata/dashboard/QWidgetMetaData.java | 83 +++++ .../frontend/QFrontendAppMetaData.java | 30 +- .../model/metadata/layout/QAppMetaData.java | 35 ++ .../PersonsByCreateDateBarChart.java | 50 +++ .../dashboard/WidgetDataLoaderTest.java | 32 ++ .../qqq/backend/core/utils/TestUtils.java | 19 +- .../module/rdbms/jdbc/QueryManager.java | 3 - .../module/rdbms/jdbc/QueryManagerTest.java | 23 ++ .../javalin/QJavalinImplementation.java | 24 ++ .../javalin/PersonsByCreateDateBarChart.java | 56 ++++ .../javalin/QJavalinImplementationTest.java | 17 + .../qqq/backend/javalin/TestUtils.java | 14 + .../sampleapp/SampleMetaDataProvider.java | 20 +- .../widgets/PersonsByCreateDateBarChart.java | 66 ++++ .../PersonsByCreateDateBarChartTest.java | 34 ++ 21 files changed, 939 insertions(+), 13 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/PersonsByCreateDateBarChart.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoaderTest.java create mode 100644 qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java create mode 100644 qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java create mode 100644 qqq-sample-project/src/test/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChartTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java index 8e7c0dfc..31192c75 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java @@ -93,4 +93,42 @@ public class QCodeLoader } } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static T getAdHoc(Class expectedType, QCodeReference codeReference) + { + if(codeReference == null) + { + return (null); + } + + if(!codeReference.getCodeType().equals(QCodeType.JAVA)) + { + /////////////////////////////////////////////////////////////////////////////////////// + // todo - 1) support more languages, 2) wrap them w/ java Functions here, 3) profit! // + /////////////////////////////////////////////////////////////////////////////////////// + throw (new IllegalArgumentException("Only JAVA code references are supported at this time.")); + } + + try + { + Class customizerClass = Class.forName(codeReference.getName()); + return ((T) customizerClass.getConstructor().newInstance()); + } + catch(Exception e) + { + LOG.error("Error initializing customizer: " + codeReference); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // return null here - under the assumption that during normal run-time operations, we'll never hit here // + // as we'll want to validate all functions in the instance validator at startup time (and IT will throw // + // if it finds an invalid code reference // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + return (null); + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java new file mode 100644 index 00000000..bf8ca097 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java @@ -0,0 +1,20 @@ +package com.kingsrook.qqq.backend.core.actions.dashboard; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class AbstractWidgetRenderer +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public abstract Object render(QInstance qInstance, QSession session) throws QException; + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java new file mode 100644 index 00000000..a0e5d214 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java @@ -0,0 +1,26 @@ +package com.kingsrook.qqq.backend.core.actions.dashboard; + + +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class WidgetDataLoader +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public Object execute(QInstance qInstance, QSession session, String name) throws QException + { + QWidgetMetaData widget = qInstance.getWidget(name); + AbstractWidgetRenderer widgetRenderer = QCodeLoader.getAdHoc(AbstractWidgetRenderer.class, widget.getCodeReference()); + return (widgetRenderer.render(qInstance, session)); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java index 96340a69..9412e0d0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java @@ -25,8 +25,8 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.io.Serializable; import java.util.List; import java.util.function.Function; -import com.kingsrook.qqq.backend.core.actions.customizers.CustomizerLoader; import com.kingsrook.qqq.backend.core.actions.customizers.Customizers; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import org.apache.logging.log4j.LogManager; @@ -62,7 +62,7 @@ public class QueryOutput extends AbstractActionOutput implements Serializable storage = new QueryOutputList(); } - postQueryRecordCustomizer = (Function) CustomizerLoader.getTableCustomizerFunction(queryInput.getTable(), Customizers.POST_QUERY_RECORD); + postQueryRecordCustomizer = (Function) QCodeLoader.getTableCustomizerFunction(queryInput.getTable(), Customizers.POST_QUERY_RECORD); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java new file mode 100644 index 00000000..f71f0c66 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java @@ -0,0 +1,299 @@ +package com.kingsrook.qqq.backend.core.model.dashboard.widgets; + + +import java.util.List; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BarChart +{ + + /* + type: "barChart", + title: "Parcel Invoice Lines per Month", + barChartData: { + labels: ["Feb 22", "Mar 22", "Apr 22", "May 22", "Jun 22", "Jul 22", "Aug 22"], + datasets: {label: "Parcel Invoice Lines", data: [50000, 22000, 11111, 22333, 40404, 9876, 2355]}, + }, + */ + + private String type = "barChart"; + private String title; + private Data barChartData; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public BarChart(String title, String seriesLabel, List labels, List data) + { + setTitle(title); + setBarChartData(new BarChart.Data() + .withLabels(labels) + .withDatasets(new BarChart.Data.DataSet() + .withLabel("Parcel Invoice Lines") + .withData(data))); + } + + + + /******************************************************************************* + ** Getter for title + ** + *******************************************************************************/ + public String getTitle() + { + return title; + } + + + + /******************************************************************************* + ** Setter for title + ** + *******************************************************************************/ + public void setTitle(String title) + { + this.title = title; + } + + + + /******************************************************************************* + ** Fluent setter for title + ** + *******************************************************************************/ + public BarChart withTitle(String title) + { + this.title = title; + return (this); + } + + + + /******************************************************************************* + ** Getter for barChartData + ** + *******************************************************************************/ + public Data getBarChartData() + { + return barChartData; + } + + + + /******************************************************************************* + ** Setter for barChartData + ** + *******************************************************************************/ + public void setBarChartData(Data barChartData) + { + this.barChartData = barChartData; + } + + + + /******************************************************************************* + ** Fluent setter for barChartData + ** + *******************************************************************************/ + public BarChart withBarChartData(Data barChartData) + { + this.barChartData = barChartData; + return (this); + } + + + + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public String getType() + { + return type; + } + + + + /******************************************************************************* + ** Setter for type + ** + *******************************************************************************/ + public void setType(String type) + { + this.type = type; + } + + + /******************************************************************************* + ** Fluent setter for type + ** + *******************************************************************************/ + public BarChart withType(String type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class Data + { + private List labels; + private DataSet datasets; + + + + /******************************************************************************* + ** Getter for labels + ** + *******************************************************************************/ + public List getLabels() + { + return labels; + } + + + + /******************************************************************************* + ** Setter for labels + ** + *******************************************************************************/ + public void setLabels(List labels) + { + this.labels = labels; + } + + + + /******************************************************************************* + ** Fluent setter for labels + ** + *******************************************************************************/ + public Data withLabels(List labels) + { + this.labels = labels; + return (this); + } + + + + /******************************************************************************* + ** Getter for datasets + ** + *******************************************************************************/ + public DataSet getDatasets() + { + return datasets; + } + + + + /******************************************************************************* + ** Setter for datasets + ** + *******************************************************************************/ + public void setDatasets(DataSet datasets) + { + this.datasets = datasets; + } + + + + /******************************************************************************* + ** Fluent setter for datasets + ** + *******************************************************************************/ + public Data withDatasets(DataSet datasets) + { + this.datasets = datasets; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class DataSet + { + private String label; + private List data; + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** Setter for label + ** + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + ** + *******************************************************************************/ + public DataSet withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for data + ** + *******************************************************************************/ + public List getData() + { + return data; + } + + + + /******************************************************************************* + ** Setter for data + ** + *******************************************************************************/ + public void setData(List data) + { + this.data = data; + } + + + + /******************************************************************************* + ** Fluent setter for data + ** + *******************************************************************************/ + public DataSet withData(List data) + { + this.data = data; + return (this); + } + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 3712d077..02718a01 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.instances.QInstanceValidationKey; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; @@ -59,6 +60,8 @@ public class QInstance private Map processes = new LinkedHashMap<>(); private Map apps = new LinkedHashMap<>(); + private Map widgets = new LinkedHashMap<>(); + // todo - lock down the object (no more changes allowed) after it's been validated? @JsonIgnore @@ -445,4 +448,60 @@ public class QInstance { this.authentication = authentication; } + + + + /******************************************************************************* + ** Getter for widgets + ** + *******************************************************************************/ + public Map getWidgets() + { + return widgets; + } + + + + /******************************************************************************* + ** Setter for widgets + ** + *******************************************************************************/ + public void setWidgets(Map widgets) + { + this.widgets = widgets; + } + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addWidget(QWidgetMetaData widget) + { + this.addWidget(widget.getName(), widget); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addWidget(String name, QWidgetMetaData widget) + { + if(this.widgets.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second widget with name: " + name)); + } + this.widgets.put(name, widget); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QWidgetMetaData getWidget(String name) + { + return (this.widgets.get(name)); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java new file mode 100644 index 00000000..0efb0dfe --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java @@ -0,0 +1,83 @@ +package com.kingsrook.qqq.backend.core.model.metadata.dashboard; + + +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QWidgetMetaData +{ + private String name; + private QCodeReference codeReference; + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Setter for name + ** + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public QWidgetMetaData withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for codeReference + ** + *******************************************************************************/ + public QCodeReference getCodeReference() + { + return codeReference; + } + + + + /******************************************************************************* + ** Setter for codeReference + ** + *******************************************************************************/ + public void setCodeReference(QCodeReference codeReference) + { + this.codeReference = codeReference; + } + + + + /******************************************************************************* + ** Fluent setter for codeReference + ** + *******************************************************************************/ + public QWidgetMetaData withCodeReference(QCodeReference codeReference) + { + this.codeReference = codeReference; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java index 6f792f97..a4bd3ba4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendAppMetaData.java @@ -26,7 +26,8 @@ import java.util.ArrayList; import java.util.List; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; /******************************************************************************* @@ -40,6 +41,7 @@ public class QFrontendAppMetaData private String label; private List children = new ArrayList<>(); + private List widgets = new ArrayList<>(); private String iconName; @@ -48,14 +50,19 @@ public class QFrontendAppMetaData /******************************************************************************* ** *******************************************************************************/ - public QFrontendAppMetaData(QAppChildMetaData appChildMetaData) + public QFrontendAppMetaData(QAppMetaData appMetaData) { - this.name = appChildMetaData.getName(); - this.label = appChildMetaData.getLabel(); + this.name = appMetaData.getName(); + this.label = appMetaData.getLabel(); - if(appChildMetaData.getIcon() != null) + if(appMetaData.getIcon() != null) { - this.iconName = appChildMetaData.getIcon().getName(); + this.iconName = appMetaData.getIcon().getName(); + } + + if(CollectionUtils.nullSafeHasContents(appMetaData.getWidgets())) + { + this.widgets = appMetaData.getWidgets(); } } @@ -127,4 +134,15 @@ public class QFrontendAppMetaData } children.add(childAppTreeNode); } + + + + /******************************************************************************* + ** Getter for widgets + ** + *******************************************************************************/ + public List getWidgets() + { + return widgets; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java index 36ac02d6..be38cb87 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java @@ -40,6 +40,7 @@ public class QAppMetaData implements QAppChildMetaData private String parentAppName; private QIcon icon; + private List widgets; /******************************************************************************* @@ -235,4 +236,38 @@ public class QAppMetaData implements QAppChildMetaData return (this); } + + + + /******************************************************************************* + ** Getter for widgets + ** + *******************************************************************************/ + public List getWidgets() + { + return widgets; + } + + + + /******************************************************************************* + ** Setter for widgets + ** + *******************************************************************************/ + public void setWidgets(List widgets) + { + this.widgets = widgets; + } + + + /******************************************************************************* + ** Fluent setter for widgets + ** + *******************************************************************************/ + public QAppMetaData withWidgets(List widgets) + { + this.widgets = widgets; + return (this); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/PersonsByCreateDateBarChart.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/PersonsByCreateDateBarChart.java new file mode 100644 index 00000000..da2c1ff7 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/PersonsByCreateDateBarChart.java @@ -0,0 +1,50 @@ +package com.kingsrook.qqq.backend.core.actions.dashboard; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.BarChart; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; + + +/******************************************************************************* + ** Sample bar chart widget + *******************************************************************************/ +public class PersonsByCreateDateBarChart extends AbstractWidgetRenderer +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Object render(QInstance qInstance, QSession session) throws QException + { + try + { + List labels = new ArrayList<>(); + List data = new ArrayList<>(); + + labels.add("Jan. 2022"); + data.add(17); + + labels.add("Feb. 2022"); + data.add(42); + + labels.add("Mar. 2022"); + data.add(47); + + labels.add("Apr. 2022"); + data.add(0); + + labels.add("May 2022"); + data.add(64); + + return (new BarChart("Persons created per Month", "Person records", labels, data)); + } + catch(Exception e) + { + throw (new QException("Error rendering widget", e)); + } + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoaderTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoaderTest.java new file mode 100644 index 00000000..c571e361 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoaderTest.java @@ -0,0 +1,32 @@ +package com.kingsrook.qqq.backend.core.actions.dashboard; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.BarChart; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for WidgetDataLoader + *******************************************************************************/ +class WidgetDataLoaderTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + Object widgetData = new WidgetDataLoader().execute(TestUtils.defineInstance(), TestUtils.getMockSession(), PersonsByCreateDateBarChart.class.getSimpleName()); + assertThat(widgetData).isInstanceOf(BarChart.class); + BarChart barChart = (BarChart) widgetData; + assertEquals("barChart", barChart.getType()); + assertThat(barChart.getTitle()).isNotBlank(); + assertNotNull(barChart.getBarChartData()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 915498d7..6b12b83b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.utils; import java.io.Serializable; import java.util.List; +import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarChart; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.GetAgeStatistics; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; @@ -38,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; @@ -115,6 +117,7 @@ public class TestUtils qInstance.addProcess(new BasicETLProcess().defineProcessMetaData()); qInstance.addProcess(new StreamedETLProcess().defineProcessMetaData()); + defineWidgets(qInstance); defineApps(qInstance); return (qInstance); @@ -122,6 +125,19 @@ public class TestUtils + + /******************************************************************************* + ** + *******************************************************************************/ + private static void defineWidgets(QInstance qInstance) + { + qInstance.addWidget(new QWidgetMetaData() + .withName(PersonsByCreateDateBarChart.class.getSimpleName()) + .withCodeReference(new QCodeReference(PersonsByCreateDateBarChart.class, null))); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -136,7 +152,8 @@ public class TestUtils .withName(APP_NAME_PEOPLE) .withChild(qInstance.getTable(TABLE_NAME_PERSON)) .withChild(qInstance.getTable(TABLE_NAME_PERSON_FILE)) - .withChild(qInstance.getApp(APP_NAME_GREETINGS))); + .withChild(qInstance.getApp(APP_NAME_GREETINGS)) + .withWidgets(List.of(PersonsByCreateDateBarChart.class.getSimpleName()))); qInstance.addApp(new QAppMetaData() .withName(APP_NAME_MISCELLANEOUS) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java index 0a90182f..ae9ac969 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java @@ -298,8 +298,6 @@ public class QueryManager *******************************************************************************/ public static List> executeStatementForRows(Connection connection, String sql, Object... params) throws SQLException { - throw (new NotImplementedException()); - /* List> rs = new ArrayList<>(); PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); @@ -318,7 +316,6 @@ public class QueryManager } return (rs); - */ } diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java index b7be5476..5930ef84 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java @@ -36,6 +36,8 @@ import java.time.LocalTime; import java.time.Month; import java.time.OffsetDateTime; import java.util.GregorianCalendar; +import java.util.List; +import java.util.Map; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -381,4 +383,25 @@ class QueryManagerTest assertEquals("Q", simpleEntity.get("CHAR_COL")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueryForRows() throws SQLException + { + Connection connection = getConnection(); + QueryManager.executeUpdate(connection, """ + INSERT INTO test_table + ( int_col, datetime_col, char_col, date_col, time_col ) + VALUES + ( 47, '2022-08-10 19:22:08', 'Q', '2022-08-10', '19:22:08') + """); + List> rows = QueryManager.executeStatementForRows(connection, "SELECT * FROM test_table"); + assertNotNull(rows); + assertEquals(47, rows.get(0).get("INT_COL")); + assertEquals("Q", rows.get(0).get("CHAR_COL")); + } + } \ No newline at end of file diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 72e0ee79..a780c379 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -36,6 +36,7 @@ import java.util.Objects; import java.util.Optional; import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager; +import com.kingsrook.qqq.backend.core.actions.dashboard.WidgetDataLoader; import com.kingsrook.qqq.backend.core.actions.metadata.MetaDataAction; import com.kingsrook.qqq.backend.core.actions.metadata.ProcessMetaDataAction; import com.kingsrook.qqq.backend.core.actions.metadata.TableMetaDataAction; @@ -278,6 +279,8 @@ public class QJavalinImplementation }); }); + get("/widget/{name}", QJavalinImplementation::widget); + //////////////////// // process routes // //////////////////// @@ -662,6 +665,27 @@ public class QJavalinImplementation + + /******************************************************************************* + ** Load the data for a widget of a given name. + *******************************************************************************/ + private static void widget(Context context) + { + try + { + InsertInput insertInput = new InsertInput(qInstance); + setupSession(context, insertInput); + + Object widgetData = new WidgetDataLoader().execute(qInstance, insertInput.getSession(), context.pathParam("name")); + context.result(JsonUtils.toJson(widgetData)); + } + catch(Exception e) + { + handleException(context, e); + } + } + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java new file mode 100644 index 00000000..09814113 --- /dev/null +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java @@ -0,0 +1,56 @@ +package com.kingsrook.qqq.backend.javalin; + + +import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.dashboard.AbstractWidgetRenderer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.BarChart; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; + + +/******************************************************************************* + ** Sample bar chart widget + *******************************************************************************/ +public class PersonsByCreateDateBarChart extends AbstractWidgetRenderer +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Object render(QInstance qInstance, QSession session) throws QException + { + try + { + ConnectionManager connectionManager = new ConnectionManager(); + Connection connection = connectionManager.getConnection(TestUtils.defineBackend()); + + List labels = new ArrayList<>(); + List data = new ArrayList<>(); + + labels.add("Jan. 2022"); + data.add(17); + + labels.add("Feb. 2022"); + data.add(42); + + labels.add("Mar. 2022"); + data.add(47); + + labels.add("Apr. 2022"); + data.add(0); + + labels.add("May 2022"); + data.add(64); + + return (new BarChart("Persons created per Month", "Person records", labels, data)); + } + catch(Exception e) + { + throw (new QException("Error rendering widget", e)); + } + } +} diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java index a7351f52..76b660d7 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java @@ -470,4 +470,21 @@ class QJavalinImplementationTest extends QJavalinTestBase assertThat(response.getBody()).contains("Unsupported report format"); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWidget() + { + HttpResponse response = Unirest.get(BASE_URL + "/widget/" + PersonsByCreateDateBarChart.class.getSimpleName()).asString(); + assertEquals(200, response.getStatus()); + JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody()); + assertNotNull(jsonObject); + assertEquals("barChart", jsonObject.getString("type")); + assertNotNull(jsonObject.getString("title")); + assertNotNull(jsonObject.getJSONObject("barChartData")); + } + } diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java index 818c2b53..831ecd25 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QValueException; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; @@ -122,11 +123,24 @@ public class TestUtils qInstance.addProcess(defineProcessSimpleSleep()); qInstance.addProcess(defineProcessScreenThenSleep()); qInstance.addProcess(defineProcessSimpleThrow()); + defineWidgets(qInstance); return (qInstance); } + /******************************************************************************* + ** + *******************************************************************************/ + private static void defineWidgets(QInstance qInstance) + { + qInstance.addWidget(new QWidgetMetaData() + .withName(PersonsByCreateDateBarChart.class.getSimpleName()) + .withCodeReference(new QCodeReference(PersonsByCreateDateBarChart.class, null))); + } + + + /******************************************************************************* ** Define the authentication used in standard tests - using 'mock' type. ** diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index 7c0e5c00..76086502 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; @@ -56,6 +57,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFor import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.sampleapp.dashboard.widgets.PersonsByCreateDateBarChart; import io.github.cdimascio.dotenv.Dotenv; @@ -114,6 +116,8 @@ public class SampleMetaDataProvider qInstance.addProcess(defineProcessScreenThenSleep()); qInstance.addProcess(defineProcessSimpleThrow()); + defineWidgets(qInstance); + defineApps(qInstance); return (qInstance); @@ -121,6 +125,18 @@ public class SampleMetaDataProvider + /******************************************************************************* + ** + *******************************************************************************/ + private static void defineWidgets(QInstance qInstance) + { + qInstance.addWidget(new QWidgetMetaData() + .withName(PersonsByCreateDateBarChart.class.getSimpleName()) + .withCodeReference(new QCodeReference(PersonsByCreateDateBarChart.class, null))); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -136,7 +152,9 @@ public class SampleMetaDataProvider .withChild(qInstance.getTable(TABLE_NAME_CITY) .withIcon(new QIcon().withName("location_city"))) .withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE)) - .withIcon(new QIcon().withName("waving_hand"))); + .withIcon(new QIcon().withName("waving_hand")) + .withWidgets(List.of(PersonsByCreateDateBarChart.class.getSimpleName())) + ); qInstance.addApp(new QAppMetaData() .withName(APP_NAME_PEOPLE) diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java new file mode 100644 index 00000000..5d7c5270 --- /dev/null +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java @@ -0,0 +1,66 @@ +package com.kingsrook.sampleapp.dashboard.widgets; + + +import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.dashboard.AbstractWidgetRenderer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.BarChart; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.sampleapp.SampleMetaDataProvider; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class PersonsByCreateDateBarChart extends AbstractWidgetRenderer +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Object render(QInstance qInstance, QSession session) throws QException + { + try + { + ConnectionManager connectionManager = new ConnectionManager(); + Connection connection = connectionManager.getConnection(SampleMetaDataProvider.defineRdbmsBackend()); + + String sql = """ + SELECT + COUNT(*) AS count, + DATE_FORMAT(create_date, '%m-%Y') AS month + FROM + person + GROUP BY + 2 + ORDER BY + 2 + """; + + List> rows = QueryManager.executeStatementForRows(connection, sql); + + List labels = new ArrayList<>(); + List data = new ArrayList<>(); + + for(Map row : rows) + { + labels.add(ValueUtils.getValueAsString(row.get("month"))); + data.add(ValueUtils.getValueAsInteger(row.get("count"))); + } + + return (new BarChart("Persons created per Month", "Person records", labels, data)); + } + catch(Exception e) + { + throw (new QException("Error rendering widget", e)); + } + } + +} diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChartTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChartTest.java new file mode 100644 index 00000000..09d8a3ae --- /dev/null +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChartTest.java @@ -0,0 +1,34 @@ +package com.kingsrook.sampleapp.dashboard.widgets; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.BarChart; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.sampleapp.SampleMetaDataProvider; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for PersonsByCreateDateBarChart + *******************************************************************************/ +class PersonsByCreateDateBarChartTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + Object widgetData = new PersonsByCreateDateBarChart().render(SampleMetaDataProvider.defineInstance(), new QSession()); + assertThat(widgetData).isInstanceOf(BarChart.class); + BarChart barChart = (BarChart) widgetData; + assertEquals("barChart", barChart.getType()); + assertThat(barChart.getTitle()).isNotBlank(); + assertNotNull(barChart.getBarChartData()); + } + +} \ No newline at end of file From 459a533f60d693b55556903cd6a49b7a252aeb68 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 22 Aug 2022 10:53:51 -0500 Subject: [PATCH 04/28] upated to work in CI (e.g., w/o database) --- .../javalin/PersonsByCreateDateBarChart.java | 5 ---- .../widgets/PersonsByCreateDateBarChart.java | 30 +++++++++++++------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java index 09814113..506b099e 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java @@ -1,7 +1,6 @@ package com.kingsrook.qqq.backend.javalin; -import java.sql.Connection; import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.actions.dashboard.AbstractWidgetRenderer; @@ -9,7 +8,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.BarChart; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.session.QSession; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; /******************************************************************************* @@ -25,9 +23,6 @@ public class PersonsByCreateDateBarChart extends AbstractWidgetRenderer { try { - ConnectionManager connectionManager = new ConnectionManager(); - Connection connection = connectionManager.getConnection(TestUtils.defineBackend()); - List labels = new ArrayList<>(); List data = new ArrayList<>(); diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java index 5d7c5270..29ec412b 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java @@ -1,19 +1,13 @@ package com.kingsrook.sampleapp.dashboard.widgets; -import java.sql.Connection; import java.util.ArrayList; import java.util.List; -import java.util.Map; import com.kingsrook.qqq.backend.core.actions.dashboard.AbstractWidgetRenderer; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.BarChart; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.session.QSession; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; -import com.kingsrook.sampleapp.SampleMetaDataProvider; /******************************************************************************* @@ -29,6 +23,8 @@ public class PersonsByCreateDateBarChart extends AbstractWidgetRenderer { try { + /* + // todo - always do this as SQL... if we had database in CI... ConnectionManager connectionManager = new ConnectionManager(); Connection connection = connectionManager.getConnection(SampleMetaDataProvider.defineRdbmsBackend()); @@ -46,14 +42,30 @@ public class PersonsByCreateDateBarChart extends AbstractWidgetRenderer List> rows = QueryManager.executeStatementForRows(connection, sql); - List labels = new ArrayList<>(); - List data = new ArrayList<>(); - for(Map row : rows) { labels.add(ValueUtils.getValueAsString(row.get("month"))); data.add(ValueUtils.getValueAsInteger(row.get("count"))); } + */ + + List labels = new ArrayList<>(); + List data = new ArrayList<>(); + + labels.add("Jan. 2022"); + data.add(17); + + labels.add("Feb. 2022"); + data.add(42); + + labels.add("Mar. 2022"); + data.add(47); + + labels.add("Apr. 2022"); + data.add(0); + + labels.add("May 2022"); + data.add(64); return (new BarChart("Persons created per Month", "Person records", labels, data)); } From c2972cd4dfc51a69e627dfaa4f144a11cbc17aa3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 23 Aug 2022 16:54:36 -0500 Subject: [PATCH 05/28] QQQ-37 checkpoint --- .../actions/async/AsyncRecordPipeLoop.java | 41 +++-- .../actions/processes/QProcessCallback.java | 10 +- .../etl/streamed/StreamedETLBackendStep.java | 101 ++---------- ...Function.java => AbstractExtractStep.java} | 25 ++- ...oadFunction.java => AbstractLoadStep.java} | 46 ++++-- ...nction.java => AbstractTransformStep.java} | 6 +- .../BaseStreamedETLStep.java | 14 +- .../ExtractViaQueryStep.java | 125 ++++++++++++++ .../LoadViaInsertStep.java | 54 +++++++ .../LoadViaUpdateStep.java | 56 +++++++ .../StreamedETLExecuteStep.java | 65 +++++--- .../StreamedETLPreviewStep.java | 31 ++-- .../StreamedETLWithFrontendProcess.java | 38 ++++- .../StreamedETLWithFrontendProcessTest.java | 152 +++++++++--------- .../qqq/backend/core/utils/TestUtils.java | 12 +- .../rdbms/actions/RDBMSUpdateAction.java | 2 +- 16 files changed, 527 insertions(+), 251 deletions(-) rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/{AbstractExtractFunction.java => AbstractExtractStep.java} (58%) rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/{AbstractLoadFunction.java => AbstractLoadStep.java} (67%) rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/{AbstractTransformFunction.java => AbstractTransformStep.java} (84%) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java index 554ec0a1..a15948f3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java @@ -12,7 +12,10 @@ import org.apache.logging.log4j.Logger; /******************************************************************************* + ** Class that knows how to Run an asynchronous job (lambda, supplier) that writes into a + ** RecordPipe, with another lambda (consumer) that consumes records from the pipe. ** + ** Takes care of the job status monitoring, blocking when the pipe is empty, etc. *******************************************************************************/ public class AsyncRecordPipeLoop { @@ -20,22 +23,32 @@ public class AsyncRecordPipeLoop private static final int TIMEOUT_AFTER_NO_RECORDS_MS = 10 * 60 * 1000; - private static final int MAX_SLEEP_MS = 1000; - private static final int INIT_SLEEP_MS = 10; + private static final int MAX_SLEEP_MS = 1000; + private static final int INIT_SLEEP_MS = 10; + private static final int MIN_RECORDS_TO_CONSUME = 10; /******************************************************************************* + ** Run an async-record-pipe-loop. ** + ** @param jobName name for the async job thread + ** @param recordLimit optionally, cancel the supplier/job after this number of records. + * e.g., for a preview step. + ** @param recordPipe constructed before this call, and used in both of the lambdas + ** @param supplier lambda that adds records into the pipe. + * e.g., a query or extract step. + ** @param consumer lambda that consumes records from the pipe + * e.g., a transform/load step. *******************************************************************************/ - public int run(String jobName, Integer recordLimit, RecordPipe recordPipe, UnsafeFunction job, UnsafeSupplier consumer) throws QException + public int run(String jobName, Integer recordLimit, RecordPipe recordPipe, UnsafeFunction supplier, UnsafeSupplier consumer) throws QException { /////////////////////////////////////////////////// // start the extraction function as an async job // /////////////////////////////////////////////////// AsyncJobManager asyncJobManager = new AsyncJobManager(); - String jobUUID = asyncJobManager.startJob(jobName, job::apply); - LOG.info("Started job [" + jobUUID + "] for record pipe streaming"); + String jobUUID = asyncJobManager.startJob(jobName, supplier::apply); + LOG.info("Started supplier job [" + jobUUID + "] for record pipe."); AsyncJobState jobState = AsyncJobState.RUNNING; AsyncJobStatus asyncJobStatus = null; @@ -47,13 +60,13 @@ public class AsyncRecordPipeLoop while(jobState.equals(AsyncJobState.RUNNING)) { - if(recordPipe.countAvailableRecords() == 0) + if(recordPipe.countAvailableRecords() < MIN_RECORDS_TO_CONSUME) { - /////////////////////////////////////////////////////////// - // if the pipe is empty, sleep to let the producer work. // - // todo - smarter sleep? like get notified vs. sleep? // - /////////////////////////////////////////////////////////// - LOG.info("No records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work"); + /////////////////////////////////////////////////////////////// + // if the pipe is too empty, sleep to let the producer work. // + // todo - smarter sleep? like get notified vs. sleep? // + /////////////////////////////////////////////////////////////// + LOG.debug("Too few records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work"); SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS); nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS); @@ -114,9 +127,9 @@ public class AsyncRecordPipeLoop throw (new QException("Job failed with an error", asyncJobStatus.getCaughtException())); } - ////////////////////////////////////////////////////// - // send the final records to transform & load steps // - ////////////////////////////////////////////////////// + //////////////////////////////////////////// + // send the final records to the consumer // + //////////////////////////////////////////// recordCount += consumer.get(); long endTime = System.currentTimeMillis(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallback.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallback.java index ce64dfad..95fe9415 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallback.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallback.java @@ -39,10 +39,16 @@ public interface QProcessCallback /******************************************************************************* ** Get the filter query for this callback. *******************************************************************************/ - QQueryFilter getQueryFilter(); + default QQueryFilter getQueryFilter() + { + return (null); + } /******************************************************************************* ** Get the field values for this callback. *******************************************************************************/ - Map getFieldValues(List fields); + default Map getFieldValues(List fields) + { + return (null); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java index ce2cfd7b..04e3c199 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java @@ -23,12 +23,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed; import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; -import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager; -import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState; -import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus; +import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; @@ -41,7 +37,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicE import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLLoadFunction; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLTransformFunction; -import com.kingsrook.qqq.backend.core.utils.SleepUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -64,6 +59,7 @@ public class StreamedETLBackendStep implements BackendStep ** *******************************************************************************/ @Override + @SuppressWarnings("checkstyle:indentation") public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { QBackendTransaction transaction = openTransaction(runBackendStepInput); @@ -74,96 +70,23 @@ public class StreamedETLBackendStep implements BackendStep BasicETLExtractFunction basicETLExtractFunction = new BasicETLExtractFunction(); basicETLExtractFunction.setRecordPipe(recordPipe); - ////////////////////////////////////////// - // run the query action as an async job // - ////////////////////////////////////////// - AsyncJobManager asyncJobManager = new AsyncJobManager(); - String queryJobUUID = asyncJobManager.startJob("StreamedETL>QueryAction", (status) -> - { - basicETLExtractFunction.run(runBackendStepInput, runBackendStepOutput); - return (runBackendStepOutput); - }); - LOG.info("Started query job [" + queryJobUUID + "] for streamed ETL"); - - AsyncJobState queryJobState = AsyncJobState.RUNNING; - AsyncJobStatus asyncJobStatus = null; - - long recordCount = 0; - int nextSleepMillis = INIT_SLEEP_MS; - long lastReceivedRecordsAt = System.currentTimeMillis(); - long jobStartTime = System.currentTimeMillis(); - - while(queryJobState.equals(AsyncJobState.RUNNING)) - { - if(recordPipe.countAvailableRecords() == 0) + //////////////////////////////////// + // run the async-record-pipe loop // + //////////////////////////////////// + int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Extract", null, recordPipe, (status) -> { - /////////////////////////////////////////////////////////// - // if the pipe is empty, sleep to let the producer work. // - // todo - smarter sleep? like get notified vs. sleep? // - /////////////////////////////////////////////////////////// - LOG.info("No records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work"); - SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS); - nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS); + basicETLExtractFunction.run(runBackendStepInput, runBackendStepOutput); + return (runBackendStepOutput); + }, + () -> (consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction)) + ); - long timeSinceLastReceivedRecord = System.currentTimeMillis() - lastReceivedRecordsAt; - if(timeSinceLastReceivedRecord > TIMEOUT_AFTER_NO_RECORDS_MS) - { - throw (new QException("Query action appears to have stopped producing records (last record received " + timeSinceLastReceivedRecord + " ms ago).")); - } - } - else - { - //////////////////////////////////////////////////////////////////////////////////////////////////////// - // if the pipe has records, consume them. reset the sleep timer so if we sleep again it'll be short. // - //////////////////////////////////////////////////////////////////////////////////////////////////////// - lastReceivedRecordsAt = System.currentTimeMillis(); - nextSleepMillis = INIT_SLEEP_MS; - - recordCount += consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction); - - LOG.info(String.format("Processed %,d records so far", recordCount)); - } - - //////////////////////////////////// - // refresh the query job's status // - //////////////////////////////////// - Optional optionalAsyncJobStatus = asyncJobManager.getJobStatus(queryJobUUID); - if(optionalAsyncJobStatus.isEmpty()) - { - ///////////////////////////////////////////////// - // todo - ... maybe some version of try-again? // - ///////////////////////////////////////////////// - throw (new QException("Could not get status of report query job [" + queryJobUUID + "]")); - } - asyncJobStatus = optionalAsyncJobStatus.get(); - queryJobState = asyncJobStatus.getState(); - } - - LOG.info("Query job [" + queryJobUUID + "] for ETL completed with status: " + asyncJobStatus); - - ///////////////////////////////////////// - // propagate errors from the query job // - ///////////////////////////////////////// - if(asyncJobStatus.getState().equals(AsyncJobState.ERROR)) - { - throw (new QException("Query job failed with an error", asyncJobStatus.getCaughtException())); - } - - ////////////////////////////////////////////////////// - // send the final records to transform & load steps // - ////////////////////////////////////////////////////// - recordCount += consumeRecordsFromPipe(recordPipe, runBackendStepInput, runBackendStepOutput, transaction); + runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount); ///////////////////// // commit the work // ///////////////////// transaction.commit(); - - long reportEndTime = System.currentTimeMillis(); - LOG.info(String.format("Processed %,d records", recordCount) - + String.format(" at end of ETL job in %,d ms (%.2f records/second).", (reportEndTime - jobStartTime), 1000d * (recordCount / (.001d + (reportEndTime - jobStartTime))))); - - runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount); } catch(Exception e) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java similarity index 58% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractFunction.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java index fbe41445..5d4b5c2a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractFunction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java @@ -3,18 +3,41 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; /******************************************************************************* + ** Base class for the Extract logic of Streamed ETL processes. ** + ** These steps are invoked by both the "preview" and the "execute" steps of a + ** StreamedETLWithFrontend process. + ** + ** Key here, is that subclasses here should put records that they're "Extracting" + ** into the recordPipe member. That is to say, DO NOT use the recordList in + ** the Step input/output objects. + ** + ** Ideally, they'll also stop once they've hit the "limit" number of records + ** (though if you keep going, the pipe will get terminated and the job will be + ** cancelled, etc...). *******************************************************************************/ -public abstract class AbstractExtractFunction implements BackendStep +public abstract class AbstractExtractStep implements BackendStep { private RecordPipe recordPipe; private Integer limit; + /******************************************************************************* + ** + *******************************************************************************/ + public Integer doCount(RunBackendStepInput runBackendStepInput) throws QException + { + return (null); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java similarity index 67% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadFunction.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java index 500ed3f6..6f70f7e4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadFunction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java @@ -3,6 +3,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.util.ArrayList; import java.util.List; +import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -11,35 +12,33 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; /******************************************************************************* + ** Base class for the Load (aka, store) logic of Streamed ETL processes. ** + ** Records are to be read out of the inputRecordPage field, and after storing, + ** should be written to the outputRecordPage. That is to say, DO NOT use the + ** recordList in the step input/output objects. + ** + ** Also - use the transaction member variable - though be aware, it *******************************************************************************/ -public abstract class AbstractLoadFunction implements BackendStep +public abstract class AbstractLoadStep implements BackendStep { - private List inputRecordPage = new ArrayList<>(); + private List inputRecordPage = new ArrayList<>(); private List outputRecordPage = new ArrayList<>(); - protected QBackendTransaction transaction; + private Optional transaction = Optional.empty(); /******************************************************************************* ** *******************************************************************************/ - public QBackendTransaction openTransaction(RunBackendStepInput runBackendStepInput) throws QException + public Optional openTransaction(RunBackendStepInput runBackendStepInput) throws QException { - this.transaction = doOpenTransaction(runBackendStepInput); - return (transaction); + return (Optional.empty()); } - /******************************************************************************* - ** - *******************************************************************************/ - protected abstract QBackendTransaction doOpenTransaction(RunBackendStepInput runBackendStepInput) throws QException; - - - /******************************************************************************* ** Getter for recordPage ** @@ -82,4 +81,25 @@ public abstract class AbstractLoadFunction implements BackendStep this.outputRecordPage = outputRecordPage; } + + + /******************************************************************************* + ** Setter for transaction + ** + *******************************************************************************/ + public void setTransaction(Optional transaction) + { + this.transaction = transaction; + } + + + + /******************************************************************************* + ** Getter for transaction + ** + *******************************************************************************/ + public Optional getTransaction() + { + return (transaction); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformFunction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java similarity index 84% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformFunction.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java index c21f4c89..8f97390c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformFunction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java @@ -8,9 +8,13 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; /******************************************************************************* + ** Base class for the Transform logic of Streamed ETL processes. ** + ** Records are to be read out of the inputRecordPage field, and after transformation, + ** should be written to the outputRecordPage. That is to say, DO NOT use the + ** recordList in the step input/output objects. *******************************************************************************/ -public abstract class AbstractTransformFunction implements BackendStep +public abstract class AbstractTransformStep implements BackendStep { private List inputRecordPage = new ArrayList<>(); private List outputRecordPage = new ArrayList<>(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java index cd239a5f..75c71501 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java @@ -11,17 +11,17 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; *******************************************************************************/ public class BaseStreamedETLStep { - protected static final int IN_MEMORY_RECORD_LIMIT = 20; + protected static final int PROCESS_OUTPUT_RECORD_LIST_LIMIT = 20; /******************************************************************************* ** *******************************************************************************/ - protected AbstractExtractFunction getExtractFunction(RunBackendStepInput runBackendStepInput) + protected AbstractExtractStep getExtractStep(RunBackendStepInput runBackendStepInput) { QCodeReference codeReference = (QCodeReference) runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_EXTRACT_CODE); - return (QCodeLoader.getBackendStep(AbstractExtractFunction.class, codeReference)); + return (QCodeLoader.getBackendStep(AbstractExtractStep.class, codeReference)); } @@ -29,10 +29,10 @@ public class BaseStreamedETLStep /******************************************************************************* ** *******************************************************************************/ - protected AbstractTransformFunction getTransformFunction(RunBackendStepInput runBackendStepInput) + protected AbstractTransformStep getTransformStep(RunBackendStepInput runBackendStepInput) { QCodeReference codeReference = (QCodeReference) runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_TRANSFORM_CODE); - return (QCodeLoader.getBackendStep(AbstractTransformFunction.class, codeReference)); + return (QCodeLoader.getBackendStep(AbstractTransformStep.class, codeReference)); } @@ -40,10 +40,10 @@ public class BaseStreamedETLStep /******************************************************************************* ** *******************************************************************************/ - protected AbstractLoadFunction getLoadFunction(RunBackendStepInput runBackendStepInput) + protected AbstractLoadStep getLoadStep(RunBackendStepInput runBackendStepInput) { QCodeReference codeReference = (QCodeReference) runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_LOAD_CODE); - return (QCodeLoader.getBackendStep(AbstractLoadFunction.class, codeReference)); + return (QCodeLoader.getBackendStep(AbstractLoadStep.class, codeReference)); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java new file mode 100644 index 00000000..f843668d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java @@ -0,0 +1,125 @@ +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.io.IOException; +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + + +/******************************************************************************* + ** Generic implementation of an ExtractStep - that runs a Query action for a + ** specified table. + ** + ** If a query is specified from the caller (e.g., using the process Callback + ** mechanism), that will be used. Else a filter (object or json) in + ** StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER will be checked. + *******************************************************************************/ +public class ExtractViaQueryStep extends AbstractExtractStep +{ + public static final String FIELD_SOURCE_TABLE = "sourceTable"; + + + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance()); + queryInput.setSession(runBackendStepInput.getSession()); + queryInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE)); + queryInput.setFilter(getQueryFilter(runBackendStepInput)); + queryInput.setRecordPipe(getRecordPipe()); + queryInput.setLimit(getLimit()); + new QueryAction().execute(queryInput); + + /////////////////////////////////////////////////////////////////// + // output is done into the pipe - so, nothing for us to do here. // + /////////////////////////////////////////////////////////////////// + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Integer doCount(RunBackendStepInput runBackendStepInput) throws QException + { + CountInput countInput = new CountInput(runBackendStepInput.getInstance()); + countInput.setSession(runBackendStepInput.getSession()); + countInput.setTableName(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE)); + countInput.setFilter(getQueryFilter(runBackendStepInput)); + CountOutput countOutput = new CountAction().execute(countInput); + return (countOutput.getCount()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected QQueryFilter getQueryFilter(RunBackendStepInput runBackendStepInput) throws QException + { + ////////////////////////////////////////////////////////////////////////////////////// + // if the queryFilterJson field is populated, read the filter from it and return it // + ////////////////////////////////////////////////////////////////////////////////////// + String queryFilterJson = runBackendStepInput.getValueString("queryFilterJson"); + if(queryFilterJson != null) + { + try + { + return (JsonUtils.toObject(queryFilterJson, QQueryFilter.class)); + } + catch(IOException e) + { + throw new QException("Error loading query filter from json field", e); + } + } + else if(runBackendStepInput.getCallback() != null && runBackendStepInput.getCallback().getQueryFilter() != null) + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else, try to get filter from process callback. if we got one, store it as a process value for later steps // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QQueryFilter queryFilter = runBackendStepInput.getCallback().getQueryFilter(); + runBackendStepInput.addValue("queryFilterJson", JsonUtils.toJson(queryFilter)); + return (queryFilter); + } + else + { + ///////////////////////////////////////////////////// + // else, see if a defaultQueryFilter was specified // + ///////////////////////////////////////////////////// + Serializable defaultQueryFilter = runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER); + if(defaultQueryFilter instanceof QQueryFilter filter) + { + return (filter); + } + if(defaultQueryFilter instanceof String string) + { + try + { + return (JsonUtils.toObject(string, QQueryFilter.class)); + } + catch(IOException e) + { + throw new QException("Error loading default query filter from json", e); + } + } + } + + throw (new QException("Could not find query filter for Extract step.")); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java new file mode 100644 index 00000000..10c0567f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java @@ -0,0 +1,54 @@ +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; + + +/******************************************************************************* + ** Generic implementation of a LoadStep - that runs an Insert action for a + ** specified table. + *******************************************************************************/ +public class LoadViaInsertStep extends AbstractLoadStep +{ + public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + + + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance()); + insertInput.setSession(runBackendStepInput.getSession()); + insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE)); + insertInput.setRecords(getInputRecordPage()); + getTransaction().ifPresent(insertInput::setTransaction); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + getOutputRecordPage().addAll(insertOutput.getRecords()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Optional openTransaction(RunBackendStepInput runBackendStepInput) throws QException + { + InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance()); + insertInput.setSession(runBackendStepInput.getSession()); + insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE)); + + return (Optional.of(new InsertAction().openTransaction(insertInput))); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java new file mode 100644 index 00000000..6c34b623 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java @@ -0,0 +1,56 @@ +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; + + +/******************************************************************************* + ** Generic implementation of a LoadStep - that runs an Update action for a + ** specified table. + *******************************************************************************/ +public class LoadViaUpdateStep extends AbstractLoadStep +{ + public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + + + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance()); + updateInput.setSession(runBackendStepInput.getSession()); + updateInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE)); + updateInput.setRecords(getInputRecordPage()); + getTransaction().ifPresent(updateInput::setTransaction); + UpdateOutput updateOutput = new UpdateAction().execute(updateInput); + getOutputRecordPage().addAll(updateOutput.getRecords()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Optional openTransaction(RunBackendStepInput runBackendStepInput) throws QException + { + InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance()); + insertInput.setSession(runBackendStepInput.getSession()); + insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE)); + + return (Optional.of(new InsertAction().openTransaction(insertInput))); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java index 2a00a5cc..f8d2f679 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.util.ArrayList; import java.util.List; +import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; @@ -36,10 +37,14 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.Str /******************************************************************************* - ** Backend step to do a execute a streamed ETL job + ** Backend step to do the execute portion of a streamed ETL job. + ** + ** Works within a transaction (per the backend module of the destination table). *******************************************************************************/ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements BackendStep { + private int currentRowCount = 1; + /******************************************************************************* @@ -49,29 +54,30 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe @SuppressWarnings("checkstyle:indentation") public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - QBackendTransaction transaction = null; + Optional transaction = Optional.empty(); try { /////////////////////////////////////////////////////// // set up the extract, transform, and load functions // /////////////////////////////////////////////////////// - RecordPipe recordPipe = new RecordPipe(); - AbstractExtractFunction extractFunction = getExtractFunction(runBackendStepInput); - extractFunction.setRecordPipe(recordPipe); + RecordPipe recordPipe = new RecordPipe(); + AbstractExtractStep extractStep = getExtractStep(runBackendStepInput); + extractStep.setRecordPipe(recordPipe); - AbstractTransformFunction transformFunction = getTransformFunction(runBackendStepInput); - AbstractLoadFunction loadFunction = getLoadFunction(runBackendStepInput); + AbstractTransformStep transformStep = getTransformStep(runBackendStepInput); + AbstractLoadStep loadStep = getLoadStep(runBackendStepInput); - transaction = loadFunction.openTransaction(runBackendStepInput); + transaction = loadStep.openTransaction(runBackendStepInput); + loadStep.setTransaction(transaction); List loadedRecordList = new ArrayList<>(); - int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Execute>ExtractFunction", null, recordPipe, (status) -> + int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Execute>ExtractStep", null, recordPipe, (status) -> { - extractFunction.run(runBackendStepInput, runBackendStepOutput); + extractStep.run(runBackendStepInput, runBackendStepOutput); return (runBackendStepOutput); }, - () -> (consumeRecordsFromPipe(recordPipe, transformFunction, loadFunction, runBackendStepInput, runBackendStepOutput, loadedRecordList)) + () -> (consumeRecordsFromPipe(recordPipe, transformStep, loadStep, runBackendStepInput, runBackendStepOutput, loadedRecordList)) ); runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount); @@ -80,9 +86,9 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe ///////////////////// // commit the work // ///////////////////// - if(transaction != null) + if(transaction.isPresent()) { - transaction.commit(); + transaction.get().commit(); } } catch(Exception e) @@ -90,9 +96,9 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe //////////////////////////////////////////////////////////////////////////////// // rollback the work, then re-throw the error for up-stream to catch & report // //////////////////////////////////////////////////////////////////////////////// - if(transaction != null) + if(transaction.isPresent()) { - transaction.rollback(); + transaction.get().rollback(); } throw (e); } @@ -101,9 +107,9 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe //////////////////////////////////////////////////////////// // always close our transactions (e.g., jdbc connections) // //////////////////////////////////////////////////////////// - if(transaction != null) + if(transaction.isPresent()) { - transaction.close(); + transaction.get().close(); } } } @@ -113,8 +119,14 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe /******************************************************************************* ** *******************************************************************************/ - private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformFunction transformFunction, AbstractLoadFunction loadFunction, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List loadedRecordList) throws QException + private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformStep transformStep, AbstractLoadStep loadStep, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List loadedRecordList) throws QException { + Integer totalRows = runBackendStepInput.getValueInteger(StreamedETLProcess.FIELD_RECORD_COUNT); + if(totalRows != null) + { + runBackendStepInput.getAsyncJobCallback().updateStatus(currentRowCount, totalRows); + } + /////////////////////////////////// // get the records from the pipe // /////////////////////////////////// @@ -123,26 +135,27 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe ///////////////////////////////////////////////////// // pass the records through the transform function // ///////////////////////////////////////////////////// - transformFunction.setInputRecordPage(qRecords); - transformFunction.setOutputRecordPage(new ArrayList<>()); - transformFunction.run(runBackendStepInput, runBackendStepOutput); + transformStep.setInputRecordPage(qRecords); + transformStep.setOutputRecordPage(new ArrayList<>()); + transformStep.run(runBackendStepInput, runBackendStepOutput); //////////////////////////////////////////////// // pass the records through the load function // //////////////////////////////////////////////// - loadFunction.setInputRecordPage(transformFunction.getOutputRecordPage()); - loadFunction.setOutputRecordPage(new ArrayList<>()); - loadFunction.run(runBackendStepInput, runBackendStepOutput); + loadStep.setInputRecordPage(transformStep.getOutputRecordPage()); + loadStep.setOutputRecordPage(new ArrayList<>()); + loadStep.run(runBackendStepInput, runBackendStepOutput); /////////////////////////////////////////////////////// // copy a small number of records to the output list // /////////////////////////////////////////////////////// int i = 0; - while(loadedRecordList.size() < IN_MEMORY_RECORD_LIMIT && i < loadFunction.getOutputRecordPage().size()) + while(loadedRecordList.size() < PROCESS_OUTPUT_RECORD_LIST_LIMIT && i < loadStep.getOutputRecordPage().size()) { - loadedRecordList.add(loadFunction.getOutputRecordPage().get(i++)); + loadedRecordList.add(loadStep.getOutputRecordPage().get(i++)); } + currentRowCount += qRecords.size(); return (qRecords.size()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java index f9cc4143..ee920b3c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; /******************************************************************************* @@ -47,20 +48,26 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe @SuppressWarnings("checkstyle:indentation") public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - RecordPipe recordPipe = new RecordPipe(); - AbstractExtractFunction extractFunction = getExtractFunction(runBackendStepInput); - extractFunction.setLimit(IN_MEMORY_RECORD_LIMIT); // todo - process field? - extractFunction.setRecordPipe(recordPipe); + RecordPipe recordPipe = new RecordPipe(); + AbstractExtractStep extractStep = getExtractStep(runBackendStepInput); + extractStep.setLimit(PROCESS_OUTPUT_RECORD_LIST_LIMIT); // todo - make this an input? + extractStep.setRecordPipe(recordPipe); - AbstractTransformFunction transformFunction = getTransformFunction(runBackendStepInput); + /////////////////////////////////////////// + // request a count from the extract step // + /////////////////////////////////////////// + Integer recordCount = extractStep.doCount(runBackendStepInput); + runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount); + + AbstractTransformStep transformStep = getTransformStep(runBackendStepInput); List transformedRecordList = new ArrayList<>(); - new AsyncRecordPipeLoop().run("StreamedETL>Preview>ExtractFunction", IN_MEMORY_RECORD_LIMIT, recordPipe, (status) -> + new AsyncRecordPipeLoop().run("StreamedETL>Preview>ExtractStep", PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> { - extractFunction.run(runBackendStepInput, runBackendStepOutput); + extractStep.run(runBackendStepInput, runBackendStepOutput); return (runBackendStepOutput); }, - () -> (consumeRecordsFromPipe(recordPipe, transformFunction, runBackendStepInput, runBackendStepOutput, transformedRecordList)) + () -> (consumeRecordsFromPipe(recordPipe, transformStep, runBackendStepInput, runBackendStepOutput, transformedRecordList)) ); runBackendStepOutput.setRecords(transformedRecordList); @@ -71,7 +78,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe /******************************************************************************* ** *******************************************************************************/ - private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformFunction transformFunction, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List transformedRecordList) throws QException + private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformStep transformStep, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List transformedRecordList) throws QException { /////////////////////////////////// // get the records from the pipe // @@ -81,13 +88,13 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe ///////////////////////////////////////////////////// // pass the records through the transform function // ///////////////////////////////////////////////////// - transformFunction.setInputRecordPage(qRecords); - transformFunction.run(runBackendStepInput, runBackendStepOutput); + transformStep.setInputRecordPage(qRecords); + transformStep.run(runBackendStepInput, runBackendStepOutput); //////////////////////////////////////////////////// // add the transformed records to the output list // //////////////////////////////////////////////////// - transformedRecordList.addAll(transformFunction.getOutputRecordPage()); + transformedRecordList.addAll(transformStep.getOutputRecordPage()); return (qRecords.size()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 4b4ba456..20d33147 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -34,6 +34,20 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; /******************************************************************************* ** Definition for Streamed ETL process that includes a frontend. ** + ** This process uses 2 backend steps, and 2 frontend steps, as follows: + ** - preview (backend) - does just a little work (limited # of rows), to give the + ** user a preview of what the final result will be - e.g., some data to seed the review screen + ** - review (frontend) - a review screen + ** - execute (backend) - processes all the rows, does all the work. + ** - result (frontend) - a result screen + ** + ** The preview & execute steps use additional BackendStep codes: + ** - Extract - gets the rows to be processed. Used in preview (but only for a + ** limited number of rows), and execute (without limit) + ** - Transform - do whatever transformation is needed to the rows. Done on preview + ** and execute. Always works with a "page" of records at a time. + ** - Load - store the records into the backend, as appropriate. Always works + ** with a "page" of records at a time. *******************************************************************************/ public class StreamedETLWithFrontendProcess { @@ -48,24 +62,31 @@ public class StreamedETLWithFrontendProcess public static final String FIELD_TRANSFORM_CODE = "transform"; public static final String FIELD_LOAD_CODE = "load"; - public static final String FIELD_SOURCE_TABLE = "sourceTable"; - public static final String FIELD_DESTINATION_TABLE = "destinationTable"; - public static final String FIELD_MAPPING_JSON = "mappingJSON"; - public static final String FIELD_RECORD_COUNT = "recordCount"; + public static final String FIELD_SOURCE_TABLE = "sourceTable"; + public static final String FIELD_DEFAULT_QUERY_FILTER = "defaultQueryFilter"; + public static final String FIELD_DESTINATION_TABLE = "destinationTable"; /******************************************************************************* ** *******************************************************************************/ - public QProcessMetaData defineProcessMetaData() + public QProcessMetaData defineProcessMetaData( + String sourceTableName, + String destinationTableName, + Class extractStepClass, + Class transformStepClass, + Class loadStepClass + ) { QStepMetaData previewStep = new QBackendStepMetaData() .withName(STEP_NAME_PREVIEW) .withCode(new QCodeReference(StreamedETLPreviewStep.class)) .withInputData(new QFunctionInputMetaData() - .withField(new QFieldMetaData().withName(FIELD_EXTRACT_CODE)) - .withField(new QFieldMetaData().withName(FIELD_TRANSFORM_CODE))); + .withField(new QFieldMetaData().withName(FIELD_SOURCE_TABLE).withDefaultValue(sourceTableName)) + .withField(new QFieldMetaData().withName(FIELD_DEFAULT_QUERY_FILTER)) + .withField(new QFieldMetaData().withName(FIELD_EXTRACT_CODE).withDefaultValue(new QCodeReference(extractStepClass))) + .withField(new QFieldMetaData().withName(FIELD_TRANSFORM_CODE).withDefaultValue(new QCodeReference(transformStepClass)))); QFrontendStepMetaData reviewStep = new QFrontendStepMetaData() .withName(STEP_NAME_REVIEW); @@ -74,7 +95,8 @@ public class StreamedETLWithFrontendProcess .withName(STEP_NAME_EXECUTE) .withCode(new QCodeReference(StreamedETLExecuteStep.class)) .withInputData(new QFunctionInputMetaData() - .withField(new QFieldMetaData().withName(FIELD_LOAD_CODE))); + .withField(new QFieldMetaData().withName(FIELD_DESTINATION_TABLE).withDefaultValue(destinationTableName)) + .withField(new QFieldMetaData().withName(FIELD_LOAD_CODE).withDefaultValue(new QCodeReference(loadStepClass)))); QFrontendStepMetaData resultStep = new QFrontendStepMetaData() .withName(STEP_NAME_RESULT); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java index 73d8792e..ad63741e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java @@ -1,23 +1,22 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; +import java.io.Serializable; import java.util.List; -import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; -import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; -import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -32,65 +31,47 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class StreamedETLWithFrontendProcessTest { + /******************************************************************************* ** *******************************************************************************/ @Test - void test() throws QException + void testSimpleSmallQueryTransformInsert() throws QException { - QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData(); - process.setTableName(TestUtils.TABLE_NAME_SHAPE); - - for(QFieldMetaData inputField : process.getInputFields()) - { - if(StreamedETLWithFrontendProcess.FIELD_EXTRACT_CODE.equals(inputField.getName())) - { - inputField.setDefaultValue(new QCodeReference(TestExtractStep.class)); - } - else if(StreamedETLWithFrontendProcess.FIELD_TRANSFORM_CODE.equals(inputField.getName())) - { - inputField.setDefaultValue(new QCodeReference(TestTransformStep.class)); - } - else if(StreamedETLWithFrontendProcess.FIELD_LOAD_CODE.equals(inputField.getName())) - { - inputField.setDefaultValue(new QCodeReference(TestLoadStep.class)); - } - } - QInstance instance = TestUtils.defineInstance(); + + //////////////////////////////////////////////////////// + // define the process - an ELT from Shapes to Persons // + //////////////////////////////////////////////////////// + QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData(TestUtils.TABLE_NAME_SHAPE, TestUtils.TABLE_NAME_PERSON, ExtractViaQueryStep.class, TestTransformStep.class, LoadViaInsertStep.class); + process.setTableName(TestUtils.TABLE_NAME_SHAPE); instance.addProcess(process); - InsertInput insertInput = new InsertInput(instance); - insertInput.setSession(TestUtils.getMockSession()); - insertInput.setTableName(TestUtils.TABLE_NAME_SHAPE); - insertInput.setRecords(List.of( - new QRecord().withTableName(TestUtils.TABLE_NAME_SHAPE).withValue("id", 1).withValue("name", "Circle"), - new QRecord().withTableName(TestUtils.TABLE_NAME_SHAPE).withValue("id", 2).withValue("name", "Triangle"), - new QRecord().withTableName(TestUtils.TABLE_NAME_SHAPE).withValue("id", 3).withValue("name", "Square") - )); - new InsertAction().execute(insertInput); + /////////////////////////////////////////////////////// + // switch the person table to use the memory backend // + /////////////////////////////////////////////////////// + instance.getTable(TestUtils.TABLE_NAME_PERSON).setBackendName(TestUtils.MEMORY_BACKEND_NAME); - List preList = TestUtils.queryTable(TestUtils.TABLE_NAME_SHAPE); + TestUtils.insertDefaultShapes(instance); + ///////////////////// + // run the process // + ///////////////////// RunProcessInput request = new RunProcessInput(instance); request.setSession(TestUtils.getMockSession()); request.setProcessName(process.getName()); request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + request.setCallback(new Callback()); RunProcessOutput result = new RunProcessAction().execute(request); assertNotNull(result); - assertTrue(result.getException().isEmpty()); - List postList = TestUtils.queryTable(TestUtils.TABLE_NAME_SHAPE); - assertEquals(6, postList.size()); + List postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_PERSON); assertThat(postList) - .anyMatch(qr -> qr.getValue("name").equals("Circle")) - .anyMatch(qr -> qr.getValue("name").equals("Triangle")) - .anyMatch(qr -> qr.getValue("name").equals("Square")) - .anyMatch(qr -> qr.getValue("name").equals("Transformed: Circle")) - .anyMatch(qr -> qr.getValue("name").equals("Transformed: Triangle")) - .anyMatch(qr -> qr.getValue("name").equals("Transformed: Square")); + .as("Should have inserted Circle").anyMatch(qr -> qr.getValue("lastName").equals("Circle")) + .as("Should have inserted Triangle").anyMatch(qr -> qr.getValue("lastName").equals("Triangle")) + .as("Should have inserted Square").anyMatch(qr -> qr.getValue("lastName").equals("Square")); } @@ -98,22 +79,34 @@ class StreamedETLWithFrontendProcessTest /******************************************************************************* ** *******************************************************************************/ - public static class TestExtractStep extends AbstractExtractFunction + @Test + void testBig() throws QException { + QInstance instance = TestUtils.defineInstance(); - /******************************************************************************* - ** Execute the backend step - using the request as input, and the result as output. - ** - *******************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance()); - queryInput.setSession(runBackendStepInput.getSession()); - queryInput.setTableName(TestUtils.TABLE_NAME_SHAPE); - queryInput.setRecordPipe(getRecordPipe()); - new QueryAction().execute(queryInput); - } + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // define the process - an ELT from Persons to Persons - using the mock backend, and set to do many many records // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_PERSON, ExtractViaQueryWithCustomLimitStep.class, TestTransformStep.class, LoadViaInsertStep.class); + process.setTableName(TestUtils.TABLE_NAME_SHAPE); + instance.addProcess(process); + + ///////////////////// + // run the process // + ///////////////////// + RunProcessInput request = new RunProcessInput(instance); + request.setSession(TestUtils.getMockSession()); + request.setProcessName(process.getName()); + request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + request.setCallback(new Callback()); + + RunProcessOutput result = new RunProcessAction().execute(request); + assertNotNull(result); + assertTrue(result.getException().isEmpty()); + + assertEquals(new ExtractViaQueryWithCustomLimitStep().getLimit(), result.getValues().get(StreamedETLProcess.FIELD_RECORD_COUNT)); + + // todo what can we assert? } @@ -121,7 +114,7 @@ class StreamedETLWithFrontendProcessTest /******************************************************************************* ** *******************************************************************************/ - public static class TestTransformStep extends AbstractTransformFunction + public static class TestTransformStep extends AbstractTransformStep { /******************************************************************************* @@ -134,8 +127,8 @@ class StreamedETLWithFrontendProcessTest for(QRecord qRecord : getInputRecordPage()) { QRecord newQRecord = new QRecord(); - newQRecord.setValue("id", null); - newQRecord.setValue("name", "Transformed: " + qRecord.getValueString("name")); + newQRecord.setValue("firstName", "Johnny"); + newQRecord.setValue("lastName", qRecord.getValueString("name")); getOutputRecordPage().add(newQRecord); } } @@ -146,34 +139,41 @@ class StreamedETLWithFrontendProcessTest /******************************************************************************* ** *******************************************************************************/ - public static class TestLoadStep extends AbstractLoadFunction + public static class Callback implements QProcessCallback { - /******************************************************************************* - ** Execute the backend step - using the request as input, and the result as output. - ** + ** Get the filter query for this callback. *******************************************************************************/ @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public QQueryFilter getQueryFilter() { - InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance()); - insertInput.setSession(runBackendStepInput.getSession()); - insertInput.setTableName(TestUtils.TABLE_NAME_SHAPE); - insertInput.setTransaction(transaction); - insertInput.setRecords(getInputRecordPage()); - new InsertAction().execute(insertInput); + return (new QQueryFilter()); } /******************************************************************************* - ** + ** Get the field values for this callback. *******************************************************************************/ @Override - protected QBackendTransaction doOpenTransaction(RunBackendStepInput runBackendStepInput) throws QException + public Map getFieldValues(List fields) { - return null; + return (null); } } + + + /******************************************************************************* + ** The Mock backend - its query action will return as many rows as the limit - + ** so let's make sure to give it a big limit. + *******************************************************************************/ + public static class ExtractViaQueryWithCustomLimitStep extends ExtractViaQueryStep + { + @Override + public Integer getLimit() + { + return (10_000); + } + } } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 9058bc4a..9984d2e3 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -440,7 +440,17 @@ public class TestUtils *******************************************************************************/ public static List queryTable(String tableName) throws QException { - QueryInput queryInput = new QueryInput(TestUtils.defineInstance()); + return (queryTable(TestUtils.defineInstance(), tableName)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static List queryTable(QInstance instance, String tableName) throws QException + { + QueryInput queryInput = new QueryInput(instance); queryInput.setSession(TestUtils.getMockSession()); queryInput.setTableName(tableName); QueryOutput queryOutput = new QueryAction().execute(queryInput); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java index 559db22c..05026f16 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java @@ -120,7 +120,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte boolean needToCloseConnection = false; if(updateInput.getTransaction() != null && updateInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction) { - LOG.debug("Using connection from insertInput [" + rdbmsTransaction.getConnection() + "]"); + LOG.debug("Using connection from updateInput [" + rdbmsTransaction.getConnection() + "]"); connection = rdbmsTransaction.getConnection(); } else From 7355a9c8aa7f255544d138df0da3db9e55c2d417 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 24 Aug 2022 10:18:59 -0500 Subject: [PATCH 06/28] QQQ-37 Updating test coverage --- .../ExtractViaQueryStep.java | 37 ++-- .../StreamedETLWithFrontendProcessTest.java | 208 ++++++++++++++---- 2 files changed, 186 insertions(+), 59 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java index f843668d..0e1b81ae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java @@ -78,14 +78,7 @@ public class ExtractViaQueryStep extends AbstractExtractStep String queryFilterJson = runBackendStepInput.getValueString("queryFilterJson"); if(queryFilterJson != null) { - try - { - return (JsonUtils.toObject(queryFilterJson, QQueryFilter.class)); - } - catch(IOException e) - { - throw new QException("Error loading query filter from json field", e); - } + return getQueryFilterFromJson(queryFilterJson, "Error loading query filter from json field"); } else if(runBackendStepInput.getCallback() != null && runBackendStepInput.getCallback().getQueryFilter() != null) { @@ -106,20 +99,30 @@ public class ExtractViaQueryStep extends AbstractExtractStep { return (filter); } - if(defaultQueryFilter instanceof String string) + else if(defaultQueryFilter instanceof String string) { - try - { - return (JsonUtils.toObject(string, QQueryFilter.class)); - } - catch(IOException e) - { - throw new QException("Error loading default query filter from json", e); - } + return getQueryFilterFromJson(string, "Error loading default query filter from json"); } } throw (new QException("Could not find query filter for Extract step.")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + private QQueryFilter getQueryFilterFromJson(String queryFilterJson, String message) throws QException + { + try + { + return (JsonUtils.toObject(queryFilterJson, QQueryFilter.class)); + } + catch(IOException e) + { + throw new QException(message, e); + } + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java index ad63741e..2c344a49 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java @@ -2,6 +2,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.io.Serializable; +import java.util.HashMap; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; @@ -11,13 +12,17 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -31,6 +36,17 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class StreamedETLWithFrontendProcessTest { + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + /******************************************************************************* ** @@ -43,7 +59,12 @@ class StreamedETLWithFrontendProcessTest //////////////////////////////////////////////////////// // define the process - an ELT from Shapes to Persons // //////////////////////////////////////////////////////// - QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData(TestUtils.TABLE_NAME_SHAPE, TestUtils.TABLE_NAME_PERSON, ExtractViaQueryStep.class, TestTransformStep.class, LoadViaInsertStep.class); + QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + TestUtils.TABLE_NAME_SHAPE, + TestUtils.TABLE_NAME_PERSON, + ExtractViaQueryStep.class, + TestTransformShapeToPersonStep.class, + LoadViaInsertStep.class); process.setTableName(TestUtils.TABLE_NAME_SHAPE); instance.addProcess(process); @@ -57,15 +78,7 @@ class StreamedETLWithFrontendProcessTest ///////////////////// // run the process // ///////////////////// - RunProcessInput request = new RunProcessInput(instance); - request.setSession(TestUtils.getMockSession()); - request.setProcessName(process.getName()); - request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); - request.setCallback(new Callback()); - - RunProcessOutput result = new RunProcessAction().execute(request); - assertNotNull(result); - assertTrue(result.getException().isEmpty()); + runProcess(instance, process); List postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_PERSON); assertThat(postList) @@ -80,33 +93,31 @@ class StreamedETLWithFrontendProcessTest ** *******************************************************************************/ @Test - void testBig() throws QException + void testSimpleSmallQueryTransformUpdate() throws QException { QInstance instance = TestUtils.defineInstance(); - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // define the process - an ELT from Persons to Persons - using the mock backend, and set to do many many records // - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_PERSON, ExtractViaQueryWithCustomLimitStep.class, TestTransformStep.class, LoadViaInsertStep.class); + //////////////////////////////////////////////////////// + // define the process - an ELT from Shapes to Shapes // + //////////////////////////////////////////////////////// + QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + TestUtils.TABLE_NAME_SHAPE, + TestUtils.TABLE_NAME_SHAPE, + ExtractViaQueryStep.class, + TestTransformUpdateShapeStep.class, + LoadViaUpdateStep.class); process.setTableName(TestUtils.TABLE_NAME_SHAPE); instance.addProcess(process); - ///////////////////// - // run the process // - ///////////////////// - RunProcessInput request = new RunProcessInput(instance); - request.setSession(TestUtils.getMockSession()); - request.setProcessName(process.getName()); - request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); - request.setCallback(new Callback()); + TestUtils.insertDefaultShapes(instance); - RunProcessOutput result = new RunProcessAction().execute(request); - assertNotNull(result); - assertTrue(result.getException().isEmpty()); + runProcess(instance, process); - assertEquals(new ExtractViaQueryWithCustomLimitStep().getLimit(), result.getValues().get(StreamedETLProcess.FIELD_RECORD_COUNT)); - - // todo what can we assert? + List postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_SHAPE); + for(String name : new String[] { "Circle", "Triangle", "Square" }) + { + assertThat(postList).as("Should have transformed and updated " + name).anyMatch(qr -> qr.getValue("name").equals("Transformed:" + name)); + } } @@ -114,7 +125,106 @@ class StreamedETLWithFrontendProcessTest /******************************************************************************* ** *******************************************************************************/ - public static class TestTransformStep extends AbstractTransformStep + @Test + void testWithDefaultQueryFilter() throws QException + { + QInstance instance = TestUtils.defineInstance(); + + //////////////////////////////////////////////////////// + // define the process - an ELT from Shapes to Shapes // + //////////////////////////////////////////////////////// + QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + TestUtils.TABLE_NAME_SHAPE, + TestUtils.TABLE_NAME_SHAPE, + ExtractViaQueryStep.class, + TestTransformUpdateShapeStep.class, + LoadViaUpdateStep.class); + process.setTableName(TestUtils.TABLE_NAME_SHAPE); + instance.addProcess(process); + + TestUtils.insertDefaultShapes(instance); + + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter().withCriteria(new QFilterCriteria("name", QCriteriaOperator.EQUALS, List.of("Square")))); + RunProcessOutput output = runProcess(instance, process, values, null); + assertEquals(1, output.getValues().get(StreamedETLProcess.FIELD_RECORD_COUNT)); + + List postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_SHAPE); + for(String name : new String[] { "Square" }) + { + assertThat(postList).as("Should have transformed and updated " + name).anyMatch(qr -> qr.getValue("name").equals("Transformed:" + name)); + } + + for(String name : new String[] { "Circle", "Triangle"}) + { + assertThat(postList).as("Should not have transformed and updated " + name).anyMatch(qr -> qr.getValue("name").equals(name)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBig() throws QException + { + QInstance instance = TestUtils.defineInstance(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // define the process - an ELT from Persons to Persons - using the mock backend, and set to do very many records // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + TestUtils.TABLE_NAME_PERSON, + TestUtils.TABLE_NAME_PERSON, + ExtractViaQueryWithCustomLimitStep.class, + TestTransformShapeToPersonStep.class, + LoadViaInsertStep.class); + process.setTableName(TestUtils.TABLE_NAME_SHAPE); + instance.addProcess(process); + + ///////////////////// + // run the process // + ///////////////////// + RunProcessOutput output = runProcess(instance, process); + assertEquals(new ExtractViaQueryWithCustomLimitStep().getLimit(), output.getValues().get(StreamedETLProcess.FIELD_RECORD_COUNT)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private RunProcessOutput runProcess(QInstance instance, QProcessMetaData process) throws QException + { + return (runProcess(instance, process, new HashMap<>(), new Callback())); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + private RunProcessOutput runProcess(QInstance instance, QProcessMetaData process, Map values, QProcessCallback callback) throws QException + { + RunProcessInput request = new RunProcessInput(instance); + request.setSession(TestUtils.getMockSession()); + request.setProcessName(process.getName()); + request.setValues(values); + request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + request.setCallback(callback); + + RunProcessOutput output = new RunProcessAction().execute(request); + assertNotNull(output); + assertTrue(output.getException().isEmpty()); + return (output); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestTransformShapeToPersonStep extends AbstractTransformStep { /******************************************************************************* @@ -136,6 +246,31 @@ class StreamedETLWithFrontendProcessTest + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestTransformUpdateShapeStep extends AbstractTransformStep + { + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + for(QRecord qRecord : getInputRecordPage()) + { + QRecord updatedQRecord = new QRecord(); + updatedQRecord.setValue("id", qRecord.getValue("id")); + updatedQRecord.setValue("name", "Transformed:" + qRecord.getValueString("name")); + getOutputRecordPage().add(updatedQRecord); + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -149,17 +284,6 @@ class StreamedETLWithFrontendProcessTest { return (new QQueryFilter()); } - - - - /******************************************************************************* - ** Get the field values for this callback. - *******************************************************************************/ - @Override - public Map getFieldValues(List fields) - { - return (null); - } } From cb22f86793fa346c1620efbf0837293ecfa2d631 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 29 Aug 2022 13:33:35 -0500 Subject: [PATCH 07/28] Checkpoint - working versions of streamed with frontend processes, with validation --- .../core/actions/customizers/QCodeLoader.java | 4 +- .../actions/customizers/TableCustomizers.java | 1 + .../actions/processes/RunProcessAction.java | 88 +++++--- .../model/actions/processes/ProcessState.java | 22 ++ .../actions/processes/ProcessSummaryLine.java | 180 +++++++++++++++ .../processes/RunBackendStepInput.java | 25 ++- .../core/model/actions/processes/Status.java | 13 ++ .../qqq/backend/core/model/data/QRecord.java | 57 ++++- .../core/model/metadata/QBackendMetaData.java | 5 +- .../core/model/metadata/QInstance.java | 21 ++ .../metadata/processes/QComponentType.java | 7 +- .../processes/QFunctionOutputMetaData.java | 7 +- .../metadata/processes/QProcessMetaData.java | 55 +++-- .../etl/basic/BasicETLProcess.java | 2 +- .../etl/streamed/StreamedETLProcess.java | 2 +- .../ProcessSummaryProviderInterface.java | 19 ++ .../StreamedETLExecuteStep.java | 14 +- .../StreamedETLPreviewStep.java | 63 +++++- .../StreamedETLValidateStep.java | 155 +++++++++++++ .../StreamedETLWithFrontendProcess.java | 90 ++++++-- .../actions/processes/RunProcessTest.java | 209 +++++++++++++++++- .../actions/reporting/ReportActionTest.java | 6 +- .../StreamedETLWithFrontendProcessTest.java | 104 ++++++++- .../qqq/backend/core/utils/TestUtils.java | 8 +- .../AbstractFilesystemBackendMetaData.java | 5 +- .../metadata/FilesystemBackendMetaData.java | 24 ++ .../BasicETLCollectSourceFileNamesStep.java | 2 +- .../s3/model/metadata/S3BackendMetaData.java | 44 +++- .../module/rdbms/jdbc/QueryManager.java | 2 + .../sampleapp/SampleMetaDataProvider.java | 65 ++++-- .../clonepeople/ClonePeopleTransformStep.java | 88 ++++++++ 31 files changed, 1253 insertions(+), 134 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ProcessSummaryProviderInterface.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java create mode 100644 qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java index 4b5fbd18..fbbc55c8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java @@ -119,7 +119,9 @@ public class QCodeLoader try { Class customizerClass = Class.forName(codeReference.getName()); - return ((T) customizerClass.getConstructor().newInstance()); + @SuppressWarnings("unchecked") + T t = (T) customizerClass.getConstructor().newInstance(); + return t; } catch(Exception e) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java index e5899fad..ef5f77b1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java @@ -24,6 +24,7 @@ public enum TableCustomizers { POST_QUERY_RECORD(new TableCustomizer("postQueryRecord", Function.class, ((Object x) -> { + @SuppressWarnings("unchecked") Function function = (Function) x; QRecord output = function.apply(new QRecord()); }))); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index 67d32747..39a9fc51 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -82,15 +82,28 @@ public class RunProcessAction runProcessOutput.setProcessUUID(runProcessInput.getProcessUUID()); UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.fromString(runProcessInput.getProcessUUID()), StateType.PROCESS_STATUS); - ProcessState processState = primeProcessState(runProcessInput, stateKey); + ProcessState processState = primeProcessState(runProcessInput, stateKey, process); - // todo - custom routing - List stepList = getAvailableStepList(process, runProcessInput); try { + String lastStepName = runProcessInput.getStartAfterStep(); + STEP_LOOP: - for(QStepMetaData step : stepList) + while(true) { + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // always refresh the step list - as any step that runs can modify it (in the process state). // + // this is why we don't do a loop over the step list - as we'd get ConcurrentModificationExceptions. // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + List stepList = getAvailableStepList(processState, process, lastStepName); + if(stepList.isEmpty()) + { + break; + } + + QStepMetaData step = stepList.get(0); + lastStepName = step.getName(); + if(step instanceof QFrontendStepMetaData) { //////////////////////////////////////////////////////////////// @@ -127,6 +140,7 @@ public class RunProcessAction /////////////////////// // Run backend steps // /////////////////////// + LOG.info("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]"); runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState); } else @@ -169,7 +183,7 @@ public class RunProcessAction ** When we start running a process (or resuming it), get data in the RunProcessRequest ** either from the state provider (if they're found, for a resume). *******************************************************************************/ - ProcessState primeProcessState(RunProcessInput runProcessInput, UUIDAndTypeStateKey stateKey) throws QException + ProcessState primeProcessState(RunProcessInput runProcessInput, UUIDAndTypeStateKey stateKey, QProcessMetaData process) throws QException { Optional optionalProcessState = loadState(stateKey); if(optionalProcessState.isEmpty()) @@ -177,11 +191,13 @@ public class RunProcessAction if(runProcessInput.getStartAfterStep() == null) { /////////////////////////////////////////////////////////////////////////////////// - // this is fine - it means its our first time running in the backend. // + // this is fine - it means it's our first time running in the backend. // // Go ahead and store the state that we have (e.g., w/ initial records & values) // /////////////////////////////////////////////////////////////////////////////////// - storeState(stateKey, runProcessInput.getProcessState()); - optionalProcessState = Optional.of(runProcessInput.getProcessState()); + ProcessState processState = runProcessInput.getProcessState(); + processState.setStepList(process.getStepList().stream().map(QStepMetaData::getName).toList()); + storeState(stateKey, processState); + optionalProcessState = Optional.of(processState); } else { @@ -249,41 +265,63 @@ public class RunProcessAction /******************************************************************************* ** Get the list of steps which are eligible to run. *******************************************************************************/ - private List getAvailableStepList(QProcessMetaData process, RunProcessInput runProcessInput) + private List getAvailableStepList(ProcessState processState, QProcessMetaData process, String lastStep) throws QException { - if(runProcessInput.getStartAfterStep() == null) + if(lastStep == null) { - ///////////////////////////////////////////////////////////////////////////// - // if the caller did not supply a 'startAfterStep', then use the full list // - ///////////////////////////////////////////////////////////////////////////// - return (process.getStepList()); + /////////////////////////////////////////////////////////////////////// + // if the caller did not supply a 'lastStep', then use the full list // + /////////////////////////////////////////////////////////////////////// + return (stepNamesToSteps(process, processState.getStepList())); } else { - //////////////////////////////////////////////////////////////////////////////// - // else, loop until the startAfterStep is found, and return the ones after it // - //////////////////////////////////////////////////////////////////////////////// - boolean foundStartAfterStep = false; - List rs = new ArrayList<>(); + //////////////////////////////////////////////////////////////////////////// + // else, loop until the 'lastStep' is found, and return the ones after it // + //////////////////////////////////////////////////////////////////////////// + boolean foundLastStep = false; + List validStepNames = new ArrayList<>(); - for(QStepMetaData step : process.getStepList()) + for(String stepName : processState.getStepList()) { - if(foundStartAfterStep) + if(foundLastStep) { - rs.add(step); + validStepNames.add(stepName); } - if(step.getName().equals(runProcessInput.getStartAfterStep())) + if(stepName.equals(lastStep)) { - foundStartAfterStep = true; + foundLastStep = true; } } - return (rs); + return (stepNamesToSteps(process, validStepNames)); } } + /******************************************************************************* + ** + *******************************************************************************/ + private List stepNamesToSteps(QProcessMetaData process, List stepNames) throws QException + { + List result = new ArrayList<>(); + + for(String stepName : stepNames) + { + QStepMetaData step = process.getStep(stepName); + if(step == null) + { + throw(new QException("Could not find a step named [" + stepName + "] in this process.")); + } + result.add(step); + } + + return (result); + } + + + /******************************************************************************* ** Load an instance of the appropriate state provider ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java index bdd87270..ccc95108 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java @@ -38,6 +38,7 @@ public class ProcessState implements Serializable { private List records = new ArrayList<>(); private Map values = new HashMap<>(); + private List stepList = new ArrayList<>(); private Optional nextStepName = Optional.empty(); @@ -117,4 +118,25 @@ public class ProcessState implements Serializable this.nextStepName = Optional.empty(); } + + + /******************************************************************************* + ** Getter for stepList + ** + *******************************************************************************/ + public List getStepList() + { + return stepList; + } + + + + /******************************************************************************* + ** Setter for stepList + ** + *******************************************************************************/ + public void setStepList(List stepList) + { + this.stepList = stepList; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java new file mode 100644 index 00000000..db627b43 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java @@ -0,0 +1,180 @@ +package com.kingsrook.qqq.backend.core.model.actions.processes; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + + +/******************************************************************************* + ** For processes that may show a review & result screen, this class provides a + ** standard way to summarize information about the records in the process. + ** + *******************************************************************************/ +public class ProcessSummaryLine implements Serializable +{ + private Status status; + private Integer count; + private String message; + + ////////////////////////////////////////////////////////////////////////// + // using ArrayList, because we need to be Serializable, and List is not // + ////////////////////////////////////////////////////////////////////////// + private ArrayList primaryKeys; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLine(Status status, Integer count, String message, ArrayList primaryKeys) + { + this.status = status; + this.count = count; + this.message = message; + this.primaryKeys = primaryKeys; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLine(Status status, Integer count, String message) + { + this.status = status; + this.count = count; + this.message = message; + } + + + + /******************************************************************************* + ** Getter for status + ** + *******************************************************************************/ + public Status getStatus() + { + return status; + } + + + + /******************************************************************************* + ** Setter for status + ** + *******************************************************************************/ + public void setStatus(Status status) + { + this.status = status; + } + + + + /******************************************************************************* + ** Getter for primaryKeys + ** + *******************************************************************************/ + public List getPrimaryKeys() + { + return primaryKeys; + } + + + + /******************************************************************************* + ** Setter for primaryKeys + ** + *******************************************************************************/ + public void setPrimaryKeys(ArrayList primaryKeys) + { + this.primaryKeys = primaryKeys; + } + + + + /******************************************************************************* + ** Getter for count + ** + *******************************************************************************/ + public Integer getCount() + { + return count; + } + + + + /******************************************************************************* + ** Setter for count + ** + *******************************************************************************/ + public void setCount(Integer count) + { + this.count = count; + } + + + + /******************************************************************************* + ** Getter for message + ** + *******************************************************************************/ + public String getMessage() + { + return message; + } + + + + /******************************************************************************* + ** Setter for message + ** + *******************************************************************************/ + public void setMessage(String message) + { + this.message = message; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void incrementCount() + { + if(count == null) + { + count = 0; + } + count++; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void incrementCountAndAddPrimaryKey(Serializable primaryKey) + { + incrementCount(); + + if(primaryKeys == null) + { + primaryKeys = new ArrayList<>(); + } + primaryKeys.add(primaryKey); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addSelfToListIfAnyCount(ArrayList rs) + { + if(count != null && count > 0) + { + rs.add(this); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index 712e1335..dd85679b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -354,7 +354,30 @@ public class RunBackendStepInput extends AbstractActionInput *******************************************************************************/ public String getValueString(String fieldName) { - return ((String) getValue(fieldName)); + return (ValueUtils.getValueAsString(getValue(fieldName))); + } + + + + /******************************************************************************* + ** Getter for a single field's value + ** + *******************************************************************************/ + public Boolean getValueBoolean(String fieldName) + { + return (ValueUtils.getValueAsBoolean(getValue(fieldName))); + } + + + + /******************************************************************************* + ** Getter for a single field's value as a primitive boolean + ** + *******************************************************************************/ + public boolean getValue_boolean(String fieldName) + { + Boolean valueAsBoolean = ValueUtils.getValueAsBoolean(getValue(fieldName)); + return (valueAsBoolean != null && valueAsBoolean); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java new file mode 100644 index 00000000..60eb77ad --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java @@ -0,0 +1,13 @@ +package com.kingsrook.qqq.backend.core.model.actions.processes; + + +/******************************************************************************* + ** Simple status enum - initially for statusesqqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java in process status lines. + *******************************************************************************/ +public enum Status +{ + OK, + WARNING, + ERROR, + INFO +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index 77cdd6e8..29903fc3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang.SerializationUtils; /******************************************************************************* @@ -87,16 +88,60 @@ public class QRecord implements Serializable /******************************************************************************* ** Copy constructor. - ** TODO ... should this do deep copies? + ** *******************************************************************************/ + @SuppressWarnings("unchecked") public QRecord(QRecord record) { this.tableName = record.tableName; this.recordLabel = record.recordLabel; - this.values = record.values; - this.displayValues = record.displayValues; - this.backendDetails = record.backendDetails; - this.errors = record.errors; + + this.values = doDeepCopy(record.values); + this.displayValues = doDeepCopy(record.displayValues); + this.backendDetails = doDeepCopy(record.backendDetails); + this.errors = doDeepCopy(record.errors); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings({ "rawtypes", "unchecked" }) + private Map doDeepCopy(Map map) + { + if(map == null) + { + return (null); + } + + if(map instanceof Serializable serializableMap) + { + return (Map) SerializationUtils.clone(serializableMap); + } + + return (new LinkedHashMap(map)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings({ "rawtypes", "unchecked" }) + private List doDeepCopy(List list) + { + if(list == null) + { + return (null); + } + + if(list instanceof Serializable serializableList) + { + return (List) SerializationUtils.clone(serializableList); + } + + return (new ArrayList(list)); } @@ -142,7 +187,6 @@ public class QRecord implements Serializable - /******************************************************************************* ** *******************************************************************************/ @@ -209,6 +253,7 @@ public class QRecord implements Serializable } + /******************************************************************************* ** Fluent setter for recordLabel ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java index ee9c53f7..ea4cd840 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java @@ -75,11 +75,10 @@ public class QBackendMetaData /******************************************************************************* ** Fluent setter, returning generically, to help sub-class fluent flows *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withName(String name) + public QBackendMetaData withName(String name) { this.name = name; - return (T) this; + return this; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 3712d077..e2d4988f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -129,6 +130,10 @@ public class QInstance *******************************************************************************/ public void addBackend(String name, QBackendMetaData backend) { + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a backend without a name.")); + } if(this.backends.containsKey(name)) { throw (new IllegalArgumentException("Attempted to add a second backend with name: " + name)); @@ -163,6 +168,10 @@ public class QInstance *******************************************************************************/ public void addTable(String name, QTableMetaData table) { + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a table without a name.")); + } if(this.tables.containsKey(name)) { throw (new IllegalArgumentException("Attempted to add a second table with name: " + name)); @@ -202,6 +211,10 @@ public class QInstance *******************************************************************************/ public void addPossibleValueSource(String name, QPossibleValueSource possibleValueSource) { + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a possibleValueSource without a name.")); + } if(this.possibleValueSources.containsKey(name)) { throw (new IllegalArgumentException("Attempted to add a second possibleValueSource with name: " + name)); @@ -252,6 +265,10 @@ public class QInstance *******************************************************************************/ public void addProcess(String name, QProcessMetaData process) { + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a process without a name.")); + } if(this.processes.containsKey(name)) { throw (new IllegalArgumentException("Attempted to add a second process with name: " + name)); @@ -286,6 +303,10 @@ public class QInstance *******************************************************************************/ public void addApp(String name, QAppMetaData app) { + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add an app without a name.")); + } if(this.apps.containsKey(name)) { throw (new IllegalArgumentException("Attempted to add a second app with name: " + name)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java index 5c0ba61e..70e6a191 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java @@ -28,7 +28,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes; public enum QComponentType { HELP_TEXT, - BULK_EDIT_FORM; + BULK_EDIT_FORM, + VALIDATION_REVIEW_SCREEN, + EDIT_FORM, + VIEW_FORM, + RECORD_LIST, + PROCESS_SUMMARY_RESULTS; /////////////////////////////////////////////////////////////////////////// // keep these values in sync with QComponentType.ts in qqq-frontend-core // /////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionOutputMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionOutputMetaData.java index 024b0f7e..7df56fa5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionOutputMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionOutputMetaData.java @@ -33,7 +33,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; *******************************************************************************/ public class QFunctionOutputMetaData { - private QRecordListMetaData recordListMetaData; + private QRecordListMetaData recordListMetaData; private List fieldList; @@ -106,11 +106,12 @@ public class QFunctionOutputMetaData + /******************************************************************************* - ** Setter for fieldList + ** Fluently add a field to the list ** *******************************************************************************/ - public QFunctionOutputMetaData addField(QFieldMetaData field) + public QFunctionOutputMetaData withField(QFieldMetaData field) { if(this.fieldList == null) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index a9fa5611..b2bd3f8c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -23,7 +23,9 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; @@ -41,7 +43,8 @@ public class QProcessMetaData implements QAppChildMetaData private String tableName; private boolean isHidden = false; - private List stepList; + private List stepList; // these are the steps that are ran, by-default, in the order they are ran in + private Map steps; // this is the full map of possible steps private String parentAppName; private QIcon icon; @@ -167,14 +170,18 @@ public class QProcessMetaData implements QAppChildMetaData *******************************************************************************/ public QProcessMetaData withStepList(List stepList) { - this.stepList = stepList; + if(stepList != null) + { + stepList.forEach(this::addStep); + } + return (this); } /******************************************************************************* - ** Setter for stepList + ** add a step to the stepList and map ** *******************************************************************************/ public QProcessMetaData addStep(QStepMetaData step) @@ -184,6 +191,30 @@ public class QProcessMetaData implements QAppChildMetaData this.stepList = new ArrayList<>(); } this.stepList.add(step); + + if(this.steps == null) + { + this.steps = new HashMap<>(); + } + this.steps.put(step.getName(), step); + + return (this); + } + + + + /******************************************************************************* + ** add a step ONLY to the step map - NOT the list w/ default execution order. + ** + *******************************************************************************/ + public QProcessMetaData addOptionalStep(QStepMetaData step) + { + if(this.steps == null) + { + this.steps = new HashMap<>(); + } + this.steps.put(step.getName(), step); + return (this); } @@ -205,15 +236,7 @@ public class QProcessMetaData implements QAppChildMetaData *******************************************************************************/ public QStepMetaData getStep(String stepName) { - for(QStepMetaData step : stepList) - { - if(step.getName().equals(stepName)) - { - return (step); - } - } - - return (null); + return (steps.get(stepName)); } @@ -245,9 +268,9 @@ public class QProcessMetaData implements QAppChildMetaData public List getInputFields() { List rs = new ArrayList<>(); - if(stepList != null) + if(steps != null) { - for(QStepMetaData step : stepList) + for(QStepMetaData step : steps.values()) { rs.addAll(step.getInputFields()); } @@ -264,9 +287,9 @@ public class QProcessMetaData implements QAppChildMetaData public List getOutputFields() { List rs = new ArrayList<>(); - if(stepList != null) + if(steps != null) { - for(QStepMetaData step : stepList) + for(QStepMetaData step : steps.values()) { rs.addAll(step.getOutputFields()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java index 1cdcb132..8dbce297 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java @@ -83,7 +83,7 @@ public class BasicETLProcess .withInputData(new QFunctionInputMetaData() .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING))) .withOutputMetaData(new QFunctionOutputMetaData() - .addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER))); + .withField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER))); return new QProcessMetaData() .withName(PROCESS_NAME) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java index 2fa636e7..bb56cf25 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java @@ -66,7 +66,7 @@ public class StreamedETLProcess .withField(new QFieldMetaData(FIELD_MAPPING_JSON, QFieldType.STRING)) .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING))) .withOutputMetaData(new QFunctionOutputMetaData() - .addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER))); + .withField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER))); return new QProcessMetaData() .withName(PROCESS_NAME) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ProcessSummaryProviderInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ProcessSummaryProviderInterface.java new file mode 100644 index 00000000..a4b74fe4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ProcessSummaryProviderInterface.java @@ -0,0 +1,19 @@ +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface ProcessSummaryProviderInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + ArrayList getProcessSummary(boolean isForResultScreen); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java index f8d2f679..3a5b1126 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java @@ -33,7 +33,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; /******************************************************************************* @@ -80,7 +79,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe () -> (consumeRecordsFromPipe(recordPipe, transformStep, loadStep, runBackendStepInput, runBackendStepOutput, loadedRecordList)) ); - runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount); + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT, recordCount); runBackendStepOutput.setRecords(loadedRecordList); ///////////////////// @@ -90,6 +89,15 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe { transaction.get().commit(); } + + if(transformStep instanceof ProcessSummaryProviderInterface processSummaryProvider) + { + ////////////////////////////////////////////////////////////////////////////////////////////// + // get the process summary from the ... transform step? the load step? each knows some... // + // TODO!! // + ////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummaryProvider.getProcessSummary(true)); + } } catch(Exception e) { @@ -121,7 +129,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe *******************************************************************************/ private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformStep transformStep, AbstractLoadStep loadStep, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List loadedRecordList) throws QException { - Integer totalRows = runBackendStepInput.getValueInteger(StreamedETLProcess.FIELD_RECORD_COUNT); + Integer totalRows = runBackendStepInput.getValueInteger(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT); if(totalRows != null) { runBackendStepInput.getAsyncJobCallback().updateStatus(currentRowCount, totalRows); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java index ee920b3c..435480ef 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -32,6 +32,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -39,6 +41,8 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.Str *******************************************************************************/ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements BackendStep { + private static final Logger LOG = LogManager.getLogger(StreamedETLPreviewStep.class); + /******************************************************************************* @@ -48,29 +52,72 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe @SuppressWarnings("checkstyle:indentation") public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - RecordPipe recordPipe = new RecordPipe(); - AbstractExtractStep extractStep = getExtractStep(runBackendStepInput); - extractStep.setLimit(PROCESS_OUTPUT_RECORD_LIST_LIMIT); // todo - make this an input? - extractStep.setRecordPipe(recordPipe); + Integer limit = PROCESS_OUTPUT_RECORD_LIST_LIMIT; // todo - use a field instead of hard-coded here? + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the do-full-validation flag has already been set, then do the validation step instead of this one // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + boolean supportsFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION); + boolean doFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION); + if(supportsFullValidation && doFullValidation) + { + skipToValidateStep(runBackendStepOutput); + return; + } /////////////////////////////////////////// // request a count from the extract step // /////////////////////////////////////////// - Integer recordCount = extractStep.doCount(runBackendStepInput); + AbstractExtractStep extractStep = getExtractStep(runBackendStepInput); + Integer recordCount = extractStep.doCount(runBackendStepInput); runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the count is less than the normal limit here, and this process supports validation, then go straight to the validation step // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // todo - maybe some future version we do this - maybe based on a user-preference + // if(supportsFullValidation && recordCount <= limit) + // { + // skipToValidateStep(runBackendStepOutput); + // return; + // } + + //////////////////////////////////////////////////////// + // proceed with a doing a limited extract & transform // + //////////////////////////////////////////////////////// + RecordPipe recordPipe = new RecordPipe(); + extractStep.setLimit(limit); + extractStep.setRecordPipe(recordPipe); + AbstractTransformStep transformStep = getTransformStep(runBackendStepInput); - List transformedRecordList = new ArrayList<>(); + List previewRecordList = new ArrayList<>(); new AsyncRecordPipeLoop().run("StreamedETL>Preview>ExtractStep", PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> { extractStep.run(runBackendStepInput, runBackendStepOutput); return (runBackendStepOutput); }, - () -> (consumeRecordsFromPipe(recordPipe, transformStep, runBackendStepInput, runBackendStepOutput, transformedRecordList)) + () -> (consumeRecordsFromPipe(recordPipe, transformStep, runBackendStepInput, runBackendStepOutput, previewRecordList)) ); - runBackendStepOutput.setRecords(transformedRecordList); + runBackendStepOutput.setRecords(previewRecordList); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void skipToValidateStep(RunBackendStepOutput runBackendStepOutput) + { + LOG.info("Skipping to validation step"); + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + ArrayList stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); + System.out.println("Step list pre: " + stepList); + stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)); + stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); + runBackendStepOutput.getProcessState().setStepList(stepList); + System.out.println("Step list post: " + stepList); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java new file mode 100644 index 00000000..d5e925d6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java @@ -0,0 +1,155 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Backend step to do a full validation of a streamed ETL job + *******************************************************************************/ +public class StreamedETLValidateStep extends BaseStreamedETLStep implements BackendStep +{ + private static final Logger LOG = LogManager.getLogger(StreamedETLValidateStep.class); + + private int currentRowCount = 1; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + @SuppressWarnings("checkstyle:indentation") + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ///////////////////////////////////////////////////////////////////// + // check if we are supported in this process - if not, return noop // + ///////////////////////////////////////////////////////////////////// + boolean supportsFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION); + if(!supportsFullValidation) + { + LOG.info("Process does not support validation, so skipping validation step"); + return; + } + + //////////////////////////////////////////////////////////////////////////////// + // check if we've been requested to run in this process - if not, return noop // + //////////////////////////////////////////////////////////////////////////////// + boolean doFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION); + if(!doFullValidation) + { + LOG.info("Not requested to do full validation, so skipping validation step"); + return; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we're proceeding with full validation, move the review step to be after validation in the step list // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + ArrayList stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); + System.out.println("Step list pre: " + stepList); + stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)); + stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); + runBackendStepOutput.getProcessState().setStepList(stepList); + System.out.println("Step list post: " + stepList); + + ////////////////////////////////////////////////////////// + // basically repeat the preview step, but with no limit // + ////////////////////////////////////////////////////////// + RecordPipe recordPipe = new RecordPipe(); + AbstractExtractStep extractStep = getExtractStep(runBackendStepInput); + extractStep.setLimit(null); + extractStep.setRecordPipe(recordPipe); + + AbstractTransformStep transformStep = getTransformStep(runBackendStepInput); + if(!(transformStep instanceof ProcessSummaryProviderInterface processSummaryProvider)) + { + throw (new QException("Transform Step " + transformStep.getClass().getName() + " does not implement ProcessSummaryProviderInterface.")); + } + + List previewRecordList = new ArrayList<>(); + int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Preview>ValidateStep", PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> + { + extractStep.run(runBackendStepInput, runBackendStepOutput); + return (runBackendStepOutput); + }, + () -> (consumeRecordsFromPipe(recordPipe, transformStep, runBackendStepInput, runBackendStepOutput, previewRecordList)) + ); + + runBackendStepOutput.setRecords(previewRecordList); + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT, recordCount); + + ////////////////////////////////////////////////////// + // get the process summary from the validation step // + ////////////////////////////////////////////////////// + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY, processSummaryProvider.getProcessSummary(false)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformStep transformStep, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List previewRecordList) throws QException + { + Integer totalRows = runBackendStepInput.getValueInteger(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT); + if(totalRows != null) + { + runBackendStepInput.getAsyncJobCallback().updateStatus(currentRowCount, totalRows); + } + + /////////////////////////////////// + // get the records from the pipe // + /////////////////////////////////// + List qRecords = recordPipe.consumeAvailableRecords(); + + ///////////////////////////////////////////////////// + // pass the records through the transform function // + ///////////////////////////////////////////////////// + transformStep.setInputRecordPage(qRecords); + transformStep.run(runBackendStepInput, runBackendStepOutput); + + /////////////////////////////////////////////////////// + // copy a small number of records to the output list // + /////////////////////////////////////////////////////// + int i = 0; + while(previewRecordList.size() < PROCESS_OUTPUT_RECORD_LIST_LIMIT && i < transformStep.getOutputRecordPage().size()) + { + previewRecordList.add(transformStep.getOutputRecordPage().get(i++)); + } + + currentRowCount += qRecords.size(); + return (qRecords.size()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 20d33147..e8b2a57a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -22,11 +22,17 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; @@ -51,60 +57,104 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; *******************************************************************************/ public class StreamedETLWithFrontendProcess { - public static final String PROCESS_NAME = "etl.streamedWithFrontend"; + public static final String STEP_NAME_PREVIEW = "preview"; + public static final String STEP_NAME_REVIEW = "review"; + public static final String STEP_NAME_VALIDATE = "validate"; + public static final String STEP_NAME_EXECUTE = "execute"; + public static final String STEP_NAME_RESULT = "result"; - public static final String STEP_NAME_PREVIEW = "preview"; - public static final String STEP_NAME_REVIEW = "review"; - public static final String STEP_NAME_EXECUTE = "execute"; - public static final String STEP_NAME_RESULT = "result"; + public static final String FIELD_EXTRACT_CODE = "extract"; // QCodeReference, of AbstractExtractStep + public static final String FIELD_TRANSFORM_CODE = "transform"; // QCodeReference, of AbstractTransformStep + public static final String FIELD_LOAD_CODE = "load"; // QCodeReference, of AbstractLoadStep - public static final String FIELD_EXTRACT_CODE = "extract"; - public static final String FIELD_TRANSFORM_CODE = "transform"; - public static final String FIELD_LOAD_CODE = "load"; + public static final String FIELD_SOURCE_TABLE = "sourceTable"; // String + public static final String FIELD_DESTINATION_TABLE = "destinationTable"; // String + public static final String FIELD_RECORD_COUNT = "recordCount"; // Integer + public static final String FIELD_DEFAULT_QUERY_FILTER = "defaultQueryFilter"; // QQueryFilter or String (json, of q QQueryFilter) - public static final String FIELD_SOURCE_TABLE = "sourceTable"; - public static final String FIELD_DEFAULT_QUERY_FILTER = "defaultQueryFilter"; - public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + public static final String FIELD_SUPPORTS_FULL_VALIDATION = "supportsFullValidation"; // Boolean + public static final String FIELD_DO_FULL_VALIDATION = "doFullValidation"; // Boolean + public static final String FIELD_VALIDATION_SUMMARY = "validationSummary"; // List + public static final String FIELD_PROCESS_SUMMARY = "processResults"; // List /******************************************************************************* ** *******************************************************************************/ - public QProcessMetaData defineProcessMetaData( + public static QProcessMetaData defineProcessMetaData( String sourceTableName, String destinationTableName, Class extractStepClass, Class transformStepClass, Class loadStepClass ) + { + Map defaultFieldValues = new HashMap<>(); + defaultFieldValues.put(FIELD_SOURCE_TABLE, sourceTableName); + defaultFieldValues.put(FIELD_DESTINATION_TABLE, destinationTableName); + return defineProcessMetaData(extractStepClass, transformStepClass, loadStepClass, defaultFieldValues); + } + + + + /******************************************************************************* + ** @param defaultFieldValues - expected to possibly contain values for the following field names: + ** - FIELD_SOURCE_TABLE + ** - FIELD_DESTINATION_TABLE + ** - FIELD_SUPPORTS_FULL_VALIDATION + ** - FIELD_DEFAULT_QUERY_FILTER + ** - FIELD_DO_FULL_VALIDATION + *******************************************************************************/ + public static QProcessMetaData defineProcessMetaData( + Class extractStepClass, + Class transformStepClass, + Class loadStepClass, + Map defaultFieldValues + ) { QStepMetaData previewStep = new QBackendStepMetaData() .withName(STEP_NAME_PREVIEW) .withCode(new QCodeReference(StreamedETLPreviewStep.class)) .withInputData(new QFunctionInputMetaData() - .withField(new QFieldMetaData().withName(FIELD_SOURCE_TABLE).withDefaultValue(sourceTableName)) - .withField(new QFieldMetaData().withName(FIELD_DEFAULT_QUERY_FILTER)) + .withField(new QFieldMetaData().withName(FIELD_SOURCE_TABLE).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE))) + .withField(new QFieldMetaData().withName(FIELD_SUPPORTS_FULL_VALIDATION).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, false))) + .withField(new QFieldMetaData().withName(FIELD_DEFAULT_QUERY_FILTER).withDefaultValue(defaultFieldValues.get(FIELD_DEFAULT_QUERY_FILTER))) .withField(new QFieldMetaData().withName(FIELD_EXTRACT_CODE).withDefaultValue(new QCodeReference(extractStepClass))) - .withField(new QFieldMetaData().withName(FIELD_TRANSFORM_CODE).withDefaultValue(new QCodeReference(transformStepClass)))); + .withField(new QFieldMetaData().withName(FIELD_TRANSFORM_CODE).withDefaultValue(new QCodeReference(transformStepClass))) + ); QFrontendStepMetaData reviewStep = new QFrontendStepMetaData() - .withName(STEP_NAME_REVIEW); + .withName(STEP_NAME_REVIEW) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VALIDATION_REVIEW_SCREEN)); + + QStepMetaData validateStep = new QBackendStepMetaData() + .withName(STEP_NAME_VALIDATE) + .withCode(new QCodeReference(StreamedETLValidateStep.class)) + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData().withName(FIELD_DO_FULL_VALIDATION).withDefaultValue(defaultFieldValues.get(FIELD_DO_FULL_VALIDATION)))) + .withOutputMetaData(new QFunctionOutputMetaData() + .withField(new QFieldMetaData().withName(FIELD_VALIDATION_SUMMARY)) + ); QStepMetaData executeStep = new QBackendStepMetaData() .withName(STEP_NAME_EXECUTE) .withCode(new QCodeReference(StreamedETLExecuteStep.class)) .withInputData(new QFunctionInputMetaData() - .withField(new QFieldMetaData().withName(FIELD_DESTINATION_TABLE).withDefaultValue(destinationTableName)) - .withField(new QFieldMetaData().withName(FIELD_LOAD_CODE).withDefaultValue(new QCodeReference(loadStepClass)))); + .withField(new QFieldMetaData().withName(FIELD_DESTINATION_TABLE).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE))) + .withField(new QFieldMetaData().withName(FIELD_LOAD_CODE).withDefaultValue(new QCodeReference(loadStepClass)))) + .withOutputMetaData(new QFunctionOutputMetaData() + .withField(new QFieldMetaData().withName(FIELD_PROCESS_SUMMARY)) + ); QFrontendStepMetaData resultStep = new QFrontendStepMetaData() - .withName(STEP_NAME_RESULT); + .withName(STEP_NAME_RESULT) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS)); return new QProcessMetaData() - .withName(PROCESS_NAME) .addStep(previewStep) .addStep(reviewStep) + .addStep(validateStep) .addStep(executeStep) .addStep(resultStep); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java index 336cb636..efa5f6c6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java @@ -23,24 +23,34 @@ package com.kingsrook.qqq.backend.core.actions.processes; import java.io.Serializable; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; import com.kingsrook.qqq.backend.core.state.StateType; import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -56,6 +66,9 @@ import static org.junit.jupiter.api.Assertions.fail; *******************************************************************************/ public class RunProcessTest { + private static final Logger LOG = LogManager.getLogger(RunProcessTest.class); + + /******************************************************************************* ** @@ -85,7 +98,7 @@ public class RunProcessTest @Test public void testBreakOnFrontendSteps() throws QException { - TestCallback callback = new TestCallback(); + TestCallback callback = new TestCallback(); QInstance instance = TestUtils.defineInstance(); RunProcessInput request = new RunProcessInput(instance); String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE; @@ -130,7 +143,7 @@ public class RunProcessTest @Test public void testSkipFrontendSteps() throws QException { - TestCallback callback = new TestCallback(); + TestCallback callback = new TestCallback(); QInstance instance = TestUtils.defineInstance(); RunProcessInput request = new RunProcessInput(instance); String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE; @@ -154,7 +167,7 @@ public class RunProcessTest @Test public void testFailOnFrontendSteps() { - TestCallback callback = new TestCallback(); + TestCallback callback = new TestCallback(); QInstance instance = TestUtils.defineInstance(); RunProcessInput request = new RunProcessInput(instance); String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE; @@ -188,7 +201,8 @@ public class RunProcessTest //////////////////////////////////////////////////////////////////////////////// RunProcessInput runProcessInput = new RunProcessInput(); UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS); - ProcessState processState = new RunProcessAction().primeProcessState(runProcessInput, stateKey); + QProcessMetaData process = TestUtils.defineInstance().getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE); + ProcessState processState = new RunProcessAction().primeProcessState(runProcessInput, stateKey, process); assertNotNull(processState); } @@ -206,10 +220,11 @@ public class RunProcessTest RunProcessInput runProcessInput = new RunProcessInput(); runProcessInput.setStartAfterStep("setupStep"); UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS); + QProcessMetaData process = TestUtils.defineInstance().getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE); assertThrows(QException.class, () -> { - new RunProcessAction().primeProcessState(runProcessInput, stateKey); + new RunProcessAction().primeProcessState(runProcessInput, stateKey, process); }); } @@ -238,7 +253,8 @@ public class RunProcessTest oldProcessState.getValues().put("foo", "fubu"); RunProcessAction.getStateProvider().put(stateKey, oldProcessState); - ProcessState primedProcessState = new RunProcessAction().primeProcessState(runProcessInput, stateKey); + QProcessMetaData process = TestUtils.defineInstance().getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE); + ProcessState primedProcessState = new RunProcessAction().primeProcessState(runProcessInput, stateKey, process); assertEquals("myValue", primedProcessState.getValues().get("key")); ///////////////////////////////////////////////////////////////////////////////////////////// @@ -250,6 +266,163 @@ public class RunProcessTest + /******************************************************************************* + ** Test a simple version of custom routing, where we just add a frontend step. + *******************************************************************************/ + @Test + void testCustomRoutingAddFrontendStep() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + + QStepMetaData back1 = new QBackendStepMetaData() + .withName("back1") + .withCode(new QCodeReference(BackendStepThatMayAddFrontendStep.class)); + + QStepMetaData front1 = new QFrontendStepMetaData() + .withName("front1"); + + String processName = "customRouting"; + qInstance.addProcess(new QProcessMetaData() + .withName(processName) + .withStepList(List.of( + back1 + ////////////////////////////////////// + // only put back1 in the step list. // + ////////////////////////////////////// + )) + .addOptionalStep(front1) + ); + + //////////////////////////////////////////////////////////// + // make sure that if we run by default, we get to the end // + //////////////////////////////////////////////////////////// + RunProcessInput request = new RunProcessInput(qInstance); + request.setSession(TestUtils.getMockSession()); + request.setProcessName(processName); + request.setCallback(new TestCallback()); + request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + RunProcessOutput result = new RunProcessAction().execute(request); + assertThat(result.getProcessState().getNextStepName()).isEmpty(); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // now run again, with the field set to cause the front1 step to be added to the step list // + ///////////////////////////////////////////////////////////////////////////////////////////// + request.addValue("shouldAddFrontendStep", true); + result = new RunProcessAction().execute(request); + assertThat(result.getProcessState().getNextStepName()).hasValue("front1"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class BackendStepThatMayAddFrontendStep implements BackendStep + { + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + LOG.info("Running " + getClass().getSimpleName()); + if(runBackendStepInput.getValue("shouldAddFrontendStep") != null) + { + List stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); + stepList.add("front1"); + runBackendStepOutput.getProcessState().setStepList(stepList); + } + } + } + + + + /******************************************************************************* + ** Test a version of custom routing, where we remove steps + *******************************************************************************/ + @Test + void testCustomRoutingRemoveSteps() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + + QStepMetaData back1 = new QBackendStepMetaData() + .withName("back1") + .withCode(new QCodeReference(BackendStepThatMayRemoveFrontendStep.class)); + + QStepMetaData front1 = new QFrontendStepMetaData() + .withName("front1"); + + QStepMetaData back2 = new QBackendStepMetaData() + .withName("back2") + .withCode(new QCodeReference(NoopBackendStep.class)); + + QStepMetaData front2 = new QFrontendStepMetaData() + .withName("front2"); + + String processName = "customRouting"; + qInstance.addProcess(new QProcessMetaData() + .withName(processName) + .withStepList(List.of( + back1, + front1, + back2, + front2 + )) + ); + + ///////////////////////////////////////////////////////////////////////////// + // make sure that if we run by default, we get stop on both frontend steps // + ///////////////////////////////////////////////////////////////////////////// + RunProcessInput request = new RunProcessInput(qInstance); + request.setSession(TestUtils.getMockSession()); + request.setProcessName(processName); + request.setCallback(new TestCallback()); + request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + RunProcessOutput result = new RunProcessAction().execute(request); + assertThat(result.getProcessState().getNextStepName()).hasValue("front1"); + + request.setStartAfterStep("front1"); + result = new RunProcessAction().execute(request); + assertThat(result.getProcessState().getNextStepName()).hasValue("front2"); + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // now run again, with the field set to cause the front1 step to be removed from the step list // + ///////////////////////////////////////////////////////////////////////////////////////////////// + request.setStartAfterStep(null); + request.addValue("shouldRemoveFrontendStep", true); + result = new RunProcessAction().execute(request); + assertThat(result.getProcessState().getNextStepName()).hasValue("front2"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class BackendStepThatMayRemoveFrontendStep implements BackendStep + { + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + LOG.info("Running " + getClass().getSimpleName()); + if(runBackendStepInput.getValue("shouldRemoveFrontendStep") != null) + { + List stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); + stepList.removeIf(s -> s.equals("front1")); + runBackendStepOutput.getProcessState().setStepList(stepList); + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -287,4 +460,24 @@ public class RunProcessTest return (rs); } } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class NoopBackendStep implements BackendStep + { + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + LOG.info("Running " + getClass().getSimpleName()); + } + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java index 71d2807f..08d0d5ba 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java @@ -64,7 +64,8 @@ class ReportActionTest runReport(recordCount, filename, ReportFormat.CSV, false); - File file = new File(filename); + File file = new File(filename); + @SuppressWarnings("unchecked") List fileLines = FileUtils.readLines(file, StandardCharsets.UTF_8.name()); assertEquals(recordCount + 1, fileLines.size()); assertTrue(file.delete()); @@ -85,7 +86,8 @@ class ReportActionTest runReport(recordCount, filename, ReportFormat.CSV, false); - File file = new File(filename); + File file = new File(filename); + @SuppressWarnings("unchecked") List fileLines = FileUtils.readLines(file, StandardCharsets.UTF_8.name()); assertEquals(recordCount + 1, fileLines.size()); assertTrue(file.delete()); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java index 2c344a49..2f1f11b6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java @@ -2,16 +2,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.io.Serializable; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -155,7 +158,7 @@ class StreamedETLWithFrontendProcessTest assertThat(postList).as("Should have transformed and updated " + name).anyMatch(qr -> qr.getValue("name").equals("Transformed:" + name)); } - for(String name : new String[] { "Circle", "Triangle"}) + for(String name : new String[] { "Circle", "Triangle" }) { assertThat(postList).as("Should not have transformed and updated " + name).anyMatch(qr -> qr.getValue("name").equals(name)); } @@ -192,6 +195,47 @@ class StreamedETLWithFrontendProcessTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWithValidationStep() throws QException + { + QInstance instance = TestUtils.defineInstance(); + + //////////////////////////////////////////////////////// + // define the process - an ELT from Shapes to Persons // + //////////////////////////////////////////////////////// + QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + TestUtils.TABLE_NAME_SHAPE, + TestUtils.TABLE_NAME_PERSON, + ExtractViaQueryStep.class, + TestTransformShapeToPersonWithValidationStep.class, + LoadViaInsertStep.class); + process.setTableName(TestUtils.TABLE_NAME_SHAPE); + instance.addProcess(process); + + /////////////////////////////////////////////////////// + // switch the person table to use the memory backend // + /////////////////////////////////////////////////////// + instance.getTable(TestUtils.TABLE_NAME_PERSON).setBackendName(TestUtils.MEMORY_BACKEND_NAME); + + TestUtils.insertDefaultShapes(instance); + + ///////////////////// + // run the process // todo - don't skip FE steps + ///////////////////// + runProcess(instance, process); + + List postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_PERSON); + assertThat(postList) + .as("Should have inserted Circle").anyMatch(qr -> qr.getValue("lastName").equals("Circle")) + .as("Should have inserted Triangle").anyMatch(qr -> qr.getValue("lastName").equals("Triangle")) + .as("Should have inserted Square").anyMatch(qr -> qr.getValue("lastName").equals("Square")); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -201,6 +245,7 @@ class StreamedETLWithFrontendProcessTest } + /******************************************************************************* ** *******************************************************************************/ @@ -246,6 +291,63 @@ class StreamedETLWithFrontendProcessTest + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestTransformShapeToPersonWithValidationStep extends AbstractTransformStep implements ProcessSummaryProviderInterface + { + private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK, 0, "can be transformed into a Person"); + private ProcessSummaryLine notAPolygonSummary = new ProcessSummaryLine(Status.OK, 0, "cannot be transformed, because they are not a Polygon"); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(boolean isForResultScreen) + { + if(isForResultScreen) + { + okSummary.setMessage("were transformed into a Person"); + } + + ArrayList summaryList = new ArrayList<>(); + summaryList.add(okSummary); + summaryList.add(notAPolygonSummary); + return (summaryList); + } + + + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + for(QRecord qRecord : getInputRecordPage()) + { + if(qRecord.getValueString("name").equals("Circle")) + { + notAPolygonSummary.incrementCountAndAddPrimaryKey(qRecord.getValue("id")); + } + else + { + QRecord newQRecord = new QRecord(); + newQRecord.setValue("firstName", "Johnny"); + newQRecord.setValue("lastName", qRecord.getValueString("name")); + getOutputRecordPage().add(newQRecord); + + okSummary.incrementCountAndAddPrimaryKey(qRecord.getValue("id")); + } + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 9984d2e3..5d093faf 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -328,7 +328,7 @@ public class TestUtils .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() .withTableName(TABLE_NAME_PERSON) - .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) + .withField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) ); @@ -366,7 +366,7 @@ public class TestUtils .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() .withTableName(TABLE_NAME_PERSON) - .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) + .withField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) ) @@ -403,7 +403,7 @@ public class TestUtils .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() .withTableName(TABLE_NAME_PERSON) - .addField(new QFieldMetaData("age", QFieldType.INTEGER))) + .withField(new QFieldMetaData("age", QFieldType.INTEGER))) .withFieldList(List.of( new QFieldMetaData("minAge", QFieldType.INTEGER), new QFieldMetaData("maxAge", QFieldType.INTEGER))))) @@ -418,7 +418,7 @@ public class TestUtils .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() .withTableName(TABLE_NAME_PERSON) - .addField(new QFieldMetaData("newAge", QFieldType.INTEGER))))); + .withField(new QFieldMetaData("newAge", QFieldType.INTEGER))))); } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java index 9002bdb8..5cf43010 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java @@ -70,11 +70,10 @@ public class AbstractFilesystemBackendMetaData extends QBackendMetaData ** Fluent setter for basePath ** *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withBasePath(String basePath) + public AbstractFilesystemBackendMetaData withBasePath(String basePath) { this.basePath = basePath; - return (T) this; + return (this); } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java index 02b3950e..51cbdb6a 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java @@ -42,4 +42,28 @@ public class FilesystemBackendMetaData extends AbstractFilesystemBackendMetaData setBackendType(FilesystemBackendModule.class); } + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + public FilesystemBackendMetaData withBasePath(String basePath) + { + setBasePath(basePath); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public FilesystemBackendMetaData withName(String name) + { + setName(name); + return this; + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java index b444a42d..fd1f56b7 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java @@ -79,6 +79,6 @@ public class BasicETLCollectSourceFileNamesStep implements BackendStep .withCodeType(QCodeType.JAVA) .withCodeUsage(QCodeUsage.BACKEND_STEP)) .withOutputMetaData(new QFunctionOutputMetaData() - .addField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING)))); + .withField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING)))); } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java index 92dfccc1..2475b6b5 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java @@ -76,11 +76,10 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData ** Fluent setter for bucketName ** *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withBucketName(String bucketName) + public S3BackendMetaData withBucketName(String bucketName) { this.bucketName = bucketName; - return (T) this; + return (this); } @@ -111,11 +110,10 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData ** Fluent setter for accessKey ** *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withAccessKey(String accessKey) + public S3BackendMetaData withAccessKey(String accessKey) { this.accessKey = accessKey; - return (T) this; + return (this); } @@ -146,11 +144,10 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData ** Fluent setter for secretKey ** *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withSecretKey(String secretKey) + public S3BackendMetaData withSecretKey(String secretKey) { this.secretKey = secretKey; - return (T) this; + return (this); } @@ -181,11 +178,10 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData ** Fluent setter for region ** *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withRegion(String region) + public S3BackendMetaData withRegion(String region) { this.region = region; - return (T) this; + return (this); } @@ -204,4 +200,28 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData secretKey = interpreter.interpret(secretKey); } + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + public S3BackendMetaData withBasePath(String basePath) + { + setBasePath(basePath); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public S3BackendMetaData withName(String name) + { + setName(name); + return this; + } + } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java index 0a90182f..ce761755 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java @@ -746,12 +746,14 @@ public class QueryManager } else if(value instanceof LocalDate ld) { + @SuppressWarnings("deprecation") java.sql.Date date = new java.sql.Date(ld.getYear() - 1900, ld.getMonthValue() - 1, ld.getDayOfMonth()); statement.setDate(index, date); return (1); } else if(value instanceof LocalTime lt) { + @SuppressWarnings("deprecation") java.sql.Time time = new java.sql.Time(lt.getHour(), lt.getMinute(), lt.getSecond()); statement.setTime(index, time); return (1); diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index 7c0e5c00..09817033 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -22,7 +22,10 @@ package com.kingsrook.sampleapp; +import java.io.Serializable; +import java.util.HashMap; import java.util.List; +import java.util.Map; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QValueException; @@ -49,6 +52,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.processes.implementations.general.LoadInitialRecordsStep; import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; @@ -56,6 +62,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFor import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.sampleapp.processes.clonepeople.ClonePeopleTransformStep; import io.github.cdimascio.dotenv.Dotenv; @@ -79,6 +86,7 @@ public class SampleMetaDataProvider public static final String PROCESS_NAME_GREET = "greet"; public static final String PROCESS_NAME_GREET_INTERACTIVE = "greetInteractive"; + public static final String PROCESS_NAME_CLONE_PEOPLE = "clonePeople"; public static final String PROCESS_NAME_SIMPLE_SLEEP = "simpleSleep"; public static final String PROCESS_NAME_SIMPLE_THROW = "simpleThrow"; public static final String PROCESS_NAME_SLEEP_INTERACTIVE = "sleepInteractive"; @@ -110,6 +118,7 @@ public class SampleMetaDataProvider qInstance.addTable(defineTableCityFile()); qInstance.addProcess(defineProcessGreetPeople()); qInstance.addProcess(defineProcessGreetPeopleInteractive()); + qInstance.addProcess(defineProcessClonePeople()); qInstance.addProcess(defineProcessSimpleSleep()); qInstance.addProcess(defineProcessScreenThenSleep()); qInstance.addProcess(defineProcessSimpleThrow()); @@ -129,19 +138,18 @@ public class SampleMetaDataProvider qInstance.addApp(new QAppMetaData() .withName(APP_NAME_GREETINGS) .withIcon(new QIcon().withName("emoji_people")) - .withChild(qInstance.getProcess(PROCESS_NAME_GREET) - .withIcon(new QIcon().withName("emoji_people"))) - .withChild(qInstance.getTable(TABLE_NAME_PERSON) - .withIcon(new QIcon().withName("person"))) - .withChild(qInstance.getTable(TABLE_NAME_CITY) - .withIcon(new QIcon().withName("location_city"))) - .withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE)) - .withIcon(new QIcon().withName("waving_hand"))); + .withChild(qInstance.getProcess(PROCESS_NAME_GREET).withIcon(new QIcon().withName("emoji_people"))) + .withChild(qInstance.getTable(TABLE_NAME_PERSON).withIcon(new QIcon().withName("person"))) + .withChild(qInstance.getTable(TABLE_NAME_CITY).withIcon(new QIcon().withName("location_city"))) + .withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE).withIcon(new QIcon().withName("waving_hand"))) + ); qInstance.addApp(new QAppMetaData() .withName(APP_NAME_PEOPLE) .withIcon(new QIcon().withName("person")) - .withChild(qInstance.getApp(APP_NAME_GREETINGS))); + .withChild(qInstance.getApp(APP_NAME_GREETINGS)) + .withChild(qInstance.getProcess(PROCESS_NAME_CLONE_PEOPLE).withIcon(new QIcon().withName("content_copy"))) + ); qInstance.addApp(new QAppMetaData() .withName(APP_NAME_MISCELLANEOUS) @@ -231,8 +239,8 @@ public class SampleMetaDataProvider .withBackendName("company_code")); table.addField(new QFieldMetaData("service_level", QFieldType.STRING) // todo PVS - .withLabel("Service Level") - .withIsRequired(true)); + .withLabel("Service Level") + .withIsRequired(true)); table.addSection(new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, List.of("id", "name"))); table.addSection(new QFieldSection("basicInfo", "Basic Info", new QIcon("dataset"), Tier.T2, List.of("company_code", "service_level"))); @@ -325,7 +333,7 @@ public class SampleMetaDataProvider .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() .withTableName(TABLE_NAME_PERSON) - .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) + .withField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) ); @@ -365,7 +373,7 @@ public class SampleMetaDataProvider .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() .withTableName(TABLE_NAME_PERSON) - .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) + .withField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) ) @@ -383,6 +391,35 @@ public class SampleMetaDataProvider + /******************************************************************************* + ** + *******************************************************************************/ + private static QProcessMetaData defineProcessClonePeople() + { + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, TABLE_NAME_PERSON); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, TABLE_NAME_PERSON); + values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true); + + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( + ExtractViaQueryStep.class, + ClonePeopleTransformStep.class, + LoadViaInsertStep.class, + values + ); + process.setName(PROCESS_NAME_CLONE_PEOPLE); + process.setTableName(TABLE_NAME_PERSON); + + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW) + .withRecordListField(new QFieldMetaData("firstName", QFieldType.STRING)) + .withRecordListField(new QFieldMetaData("lastName", QFieldType.STRING)) + ; + + return (process); + } + + + /******************************************************************************* ** Define a process with just one step that sleeps *******************************************************************************/ @@ -467,7 +504,7 @@ public class SampleMetaDataProvider .withCodeType(QCodeType.JAVA) .withCodeUsage(QCodeUsage.BACKEND_STEP)) .withInputData(new QFunctionInputMetaData() - .addField(new QFieldMetaData(SleeperStep.FIELD_SLEEP_MILLIS, QFieldType.INTEGER)))); + .withField(new QFieldMetaData(SleeperStep.FIELD_SLEEP_MILLIS, QFieldType.INTEGER)))); } } diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java new file mode 100644 index 00000000..a8874be4 --- /dev/null +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java @@ -0,0 +1,88 @@ +package com.kingsrook.sampleapp.processes.clonepeople; + + +import java.io.Serializable; +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ClonePeopleTransformStep extends AbstractTransformStep implements ProcessSummaryProviderInterface +{ + private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK, 0, "can be cloned with no issues."); + private ProcessSummaryLine warningCloneSummary = new ProcessSummaryLine(Status.WARNING, 0, "can be cloned, but because are already a clone, their clone cannot be cloned in the future."); + private ProcessSummaryLine refuseCloningSummary = new ProcessSummaryLine(Status.ERROR, 0, "say they don't want to be cloned (probably a Garret...)"); + private ProcessSummaryLine nestedCloneSummary = new ProcessSummaryLine(Status.ERROR, 0, "are already a clone of a clone, so they can't be cloned again."); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(boolean isForResultScreen) + { + if(isForResultScreen) + { + okSummary.setMessage("were cloned"); + warningCloneSummary.setMessage("were already a clone, so they were cloned again now, but their clones cannot be cloned after this."); + nestedCloneSummary.setMessage("are already a clone of a clone, so they weren't cloned again."); + } + + ArrayList rs = new ArrayList<>(); + okSummary.addSelfToListIfAnyCount(rs); + warningCloneSummary.addSelfToListIfAnyCount(rs); + refuseCloningSummary.addSelfToListIfAnyCount(rs); + nestedCloneSummary.addSelfToListIfAnyCount(rs); + return (rs); + } + + + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + for(QRecord inputPerson : getInputRecordPage()) + { + Serializable id = inputPerson.getValue("id"); + if("Garret".equals(inputPerson.getValueString("firstName"))) + { + refuseCloningSummary.incrementCountAndAddPrimaryKey(id); + } + else if(inputPerson.getValueString("firstName").matches("Clone of.*Clone of.*")) + { + nestedCloneSummary.incrementCountAndAddPrimaryKey(id); + } + else + { + QRecord outputPerson = new QRecord(inputPerson); + outputPerson.setValue("id", null); + outputPerson.setValue("firstName", "Clone of: " + inputPerson.getValueString("firstName")); + getOutputRecordPage().add(outputPerson); + + if(inputPerson.getValueString("firstName").matches("Clone of.*")) + { + warningCloneSummary.incrementCountAndAddPrimaryKey(id); + } + else + { + okSummary.incrementCountAndAddPrimaryKey(id); + } + } + } + } + +} From 39f065e23e197da6460fc511b60c24ee5296a470 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 29 Aug 2022 13:52:54 -0500 Subject: [PATCH 08/28] Fixing tests --- .../actions/processes/ProcessSummaryLine.java | 11 +++ .../StreamedETLWithFrontendProcessTest.java | 77 +++++++++++++++---- 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java index db627b43..ac94faff 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java @@ -49,6 +49,17 @@ public class ProcessSummaryLine implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return "ProcessSummaryLine{status=" + status + ", count=" + count + ", message='" + message + '\'' + '}'; + } + + + /******************************************************************************* ** Getter for status ** diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java index 2f1f11b6..7650099f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java @@ -62,12 +62,13 @@ class StreamedETLWithFrontendProcessTest //////////////////////////////////////////////////////// // define the process - an ELT from Shapes to Persons // //////////////////////////////////////////////////////// - QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( TestUtils.TABLE_NAME_SHAPE, TestUtils.TABLE_NAME_PERSON, ExtractViaQueryStep.class, TestTransformShapeToPersonStep.class, LoadViaInsertStep.class); + process.setName("test"); process.setTableName(TestUtils.TABLE_NAME_SHAPE); instance.addProcess(process); @@ -103,12 +104,13 @@ class StreamedETLWithFrontendProcessTest //////////////////////////////////////////////////////// // define the process - an ELT from Shapes to Shapes // //////////////////////////////////////////////////////// - QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( TestUtils.TABLE_NAME_SHAPE, TestUtils.TABLE_NAME_SHAPE, ExtractViaQueryStep.class, TestTransformUpdateShapeStep.class, LoadViaUpdateStep.class); + process.setName("test"); process.setTableName(TestUtils.TABLE_NAME_SHAPE); instance.addProcess(process); @@ -136,12 +138,13 @@ class StreamedETLWithFrontendProcessTest //////////////////////////////////////////////////////// // define the process - an ELT from Shapes to Shapes // //////////////////////////////////////////////////////// - QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( TestUtils.TABLE_NAME_SHAPE, TestUtils.TABLE_NAME_SHAPE, ExtractViaQueryStep.class, TestTransformUpdateShapeStep.class, LoadViaUpdateStep.class); + process.setName("test"); process.setTableName(TestUtils.TABLE_NAME_SHAPE); instance.addProcess(process); @@ -177,12 +180,13 @@ class StreamedETLWithFrontendProcessTest /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // define the process - an ELT from Persons to Persons - using the mock backend, and set to do very many records // /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_PERSON, ExtractViaQueryWithCustomLimitStep.class, TestTransformShapeToPersonStep.class, LoadViaInsertStep.class); + process.setName("test"); process.setTableName(TestUtils.TABLE_NAME_SHAPE); instance.addProcess(process); @@ -206,12 +210,13 @@ class StreamedETLWithFrontendProcessTest //////////////////////////////////////////////////////// // define the process - an ELT from Shapes to Persons // //////////////////////////////////////////////////////// - QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( TestUtils.TABLE_NAME_SHAPE, TestUtils.TABLE_NAME_PERSON, ExtractViaQueryStep.class, TestTransformShapeToPersonWithValidationStep.class, LoadViaInsertStep.class); + process.setName("test"); process.setTableName(TestUtils.TABLE_NAME_SHAPE); instance.addProcess(process); @@ -222,14 +227,47 @@ class StreamedETLWithFrontendProcessTest TestUtils.insertDefaultShapes(instance); - ///////////////////// - // run the process // todo - don't skip FE steps - ///////////////////// - runProcess(instance, process); + /////////////////////////////////////////////////////////////////////////// + // run the process - breaking on the first instance of the Review screen // + /////////////////////////////////////////////////////////////////////////// + RunProcessOutput runProcessOutput = runProcess(instance, process, Map.of(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true), new Callback(), RunProcessInput.FrontendStepBehavior.BREAK); + assertThat(runProcessOutput.getProcessState().getNextStepName()).hasValue("review"); + //////////////////////////////////////////////////////// + // continue the process - telling it to do validation // + //////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(instance); + runProcessInput.setSession(TestUtils.getMockSession()); + runProcessInput.setProcessName(process.getName()); + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + runProcessInput.setProcessUUID(runProcessOutput.getProcessUUID()); + runProcessInput.setStartAfterStep(runProcessOutput.getProcessState().getNextStepName().get()); + runProcessInput.setValues(Map.of(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true)); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertNotNull(runProcessOutput); + assertTrue(runProcessOutput.getException().isEmpty()); + assertThat(runProcessOutput.getProcessState().getNextStepName()).hasValue("review"); + + List validationSummaryLines = (List) runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY); + assertThat(validationSummaryLines) + .usingRecursiveFieldByFieldElementComparatorOnFields("status", "count") + .contains(new ProcessSummaryLine(Status.OK, 2, null)) + .contains(new ProcessSummaryLine(Status.ERROR, 1, null)); + + /////////////////////////////////////////////////////// + // continue the process - going to the result screen // + /////////////////////////////////////////////////////// + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertNotNull(runProcessOutput); + assertTrue(runProcessOutput.getException().isEmpty()); + assertThat(runProcessOutput.getProcessState().getNextStepName()).hasValue("result"); + + //////////////////////////////////// + // query for the inserted records // + //////////////////////////////////// List postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_PERSON); assertThat(postList) - .as("Should have inserted Circle").anyMatch(qr -> qr.getValue("lastName").equals("Circle")) + .as("Should not have inserted Circle").noneMatch(qr -> qr.getValue("lastName").equals("Circle")) .as("Should have inserted Triangle").anyMatch(qr -> qr.getValue("lastName").equals("Triangle")) .as("Should have inserted Square").anyMatch(qr -> qr.getValue("lastName").equals("Square")); } @@ -250,12 +288,25 @@ class StreamedETLWithFrontendProcessTest ** *******************************************************************************/ private RunProcessOutput runProcess(QInstance instance, QProcessMetaData process, Map values, QProcessCallback callback) throws QException + { + return (runProcess(instance, process, values, callback, RunProcessInput.FrontendStepBehavior.SKIP)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private RunProcessOutput runProcess(QInstance instance, QProcessMetaData process, Map values, QProcessCallback callback, RunProcessInput.FrontendStepBehavior frontendStepBehavior) throws QException { RunProcessInput request = new RunProcessInput(instance); request.setSession(TestUtils.getMockSession()); request.setProcessName(process.getName()); - request.setValues(values); - request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + ////////////////////////////////////////////////////////////// + // wrap the map here, in case it was given as un-modifiable // + ////////////////////////////////////////////////////////////// + request.setValues(new HashMap<>(values)); + request.setFrontendStepBehavior(frontendStepBehavior); request.setCallback(callback); RunProcessOutput output = new RunProcessAction().execute(request); @@ -297,7 +348,7 @@ class StreamedETLWithFrontendProcessTest public static class TestTransformShapeToPersonWithValidationStep extends AbstractTransformStep implements ProcessSummaryProviderInterface { private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK, 0, "can be transformed into a Person"); - private ProcessSummaryLine notAPolygonSummary = new ProcessSummaryLine(Status.OK, 0, "cannot be transformed, because they are not a Polygon"); + private ProcessSummaryLine notAPolygonSummary = new ProcessSummaryLine(Status.ERROR, 0, "cannot be transformed, because they are not a Polygon"); From 9106b825601985405fbb9a02173c25046ee7aacd Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 29 Aug 2022 14:12:09 -0500 Subject: [PATCH 09/28] Fixing tests --- .../actions/processes/ProcessSummaryLine.java | 12 +++++++++++ .../StreamedETLWithFrontendProcess.java | 21 ++++++++++--------- .../qqq/frontend/picocli/QCommandBuilder.java | 11 +++++++++- .../sampleapp/SampleMetaDataProvider.java | 12 +++++++---- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java index ac94faff..21c5db44 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java @@ -49,6 +49,18 @@ public class ProcessSummaryLine implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLine(Status status, String message) + { + this.status = status; + this.message = message; + this.count = 0; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index e8b2a57a..292dee84 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.Map; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; @@ -117,11 +118,11 @@ public class StreamedETLWithFrontendProcess .withName(STEP_NAME_PREVIEW) .withCode(new QCodeReference(StreamedETLPreviewStep.class)) .withInputData(new QFunctionInputMetaData() - .withField(new QFieldMetaData().withName(FIELD_SOURCE_TABLE).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE))) - .withField(new QFieldMetaData().withName(FIELD_SUPPORTS_FULL_VALIDATION).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, false))) - .withField(new QFieldMetaData().withName(FIELD_DEFAULT_QUERY_FILTER).withDefaultValue(defaultFieldValues.get(FIELD_DEFAULT_QUERY_FILTER))) - .withField(new QFieldMetaData().withName(FIELD_EXTRACT_CODE).withDefaultValue(new QCodeReference(extractStepClass))) - .withField(new QFieldMetaData().withName(FIELD_TRANSFORM_CODE).withDefaultValue(new QCodeReference(transformStepClass))) + .withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE))) + .withField(new QFieldMetaData(FIELD_SUPPORTS_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, false))) + .withField(new QFieldMetaData(FIELD_DEFAULT_QUERY_FILTER, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DEFAULT_QUERY_FILTER))) + .withField(new QFieldMetaData(FIELD_EXTRACT_CODE, QFieldType.STRING).withDefaultValue(new QCodeReference(extractStepClass))) + .withField(new QFieldMetaData(FIELD_TRANSFORM_CODE, QFieldType.STRING).withDefaultValue(new QCodeReference(transformStepClass))) ); QFrontendStepMetaData reviewStep = new QFrontendStepMetaData() @@ -132,19 +133,19 @@ public class StreamedETLWithFrontendProcess .withName(STEP_NAME_VALIDATE) .withCode(new QCodeReference(StreamedETLValidateStep.class)) .withInputData(new QFunctionInputMetaData() - .withField(new QFieldMetaData().withName(FIELD_DO_FULL_VALIDATION).withDefaultValue(defaultFieldValues.get(FIELD_DO_FULL_VALIDATION)))) + .withField(new QFieldMetaData(FIELD_DO_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.get(FIELD_DO_FULL_VALIDATION)))) .withOutputMetaData(new QFunctionOutputMetaData() - .withField(new QFieldMetaData().withName(FIELD_VALIDATION_SUMMARY)) + .withField(new QFieldMetaData(FIELD_VALIDATION_SUMMARY, QFieldType.STRING)) ); QStepMetaData executeStep = new QBackendStepMetaData() .withName(STEP_NAME_EXECUTE) .withCode(new QCodeReference(StreamedETLExecuteStep.class)) .withInputData(new QFunctionInputMetaData() - .withField(new QFieldMetaData().withName(FIELD_DESTINATION_TABLE).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE))) - .withField(new QFieldMetaData().withName(FIELD_LOAD_CODE).withDefaultValue(new QCodeReference(loadStepClass)))) + .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE))) + .withField(new QFieldMetaData(FIELD_LOAD_CODE, QFieldType.STRING).withDefaultValue(new QCodeReference(loadStepClass)))) .withOutputMetaData(new QFunctionOutputMetaData() - .withField(new QFieldMetaData().withName(FIELD_PROCESS_SUMMARY)) + .withField(new QFieldMetaData(FIELD_PROCESS_SUMMARY, QFieldType.STRING)) ); QFrontendStepMetaData resultStep = new QFrontendStepMetaData() diff --git a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java index b1e18017..fe8955eb 100644 --- a/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java +++ b/qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QCommandBuilder.java @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; @@ -384,7 +385,7 @@ public class QCommandBuilder for(QFieldMetaData field : inputFieldMap.values()) { processCommand.addOption(CommandLine.Model.OptionSpec.builder("--field-" + field.getName()) - .type(getClassForField(field)) + .type(Objects.requireNonNullElse(getClassForField(field), String.class)) .build()); } @@ -401,6 +402,14 @@ public class QCommandBuilder @SuppressWarnings("checkstyle:Indentation") private Class getClassForField(QFieldMetaData field) { + if(field.getType() == null) + { + /////////////////////////////////////////////////// + // shouldn't happen, but just in case, avoid NPE // + /////////////////////////////////////////////////// + return (null); + } + // @formatter:off // IJ can't do new-style switch correctly yet... return switch(field.getType()) { diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index 09817033..ce773b14 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -43,6 +43,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; @@ -320,10 +322,7 @@ public class SampleMetaDataProvider .withIsHidden(true) .addStep(new QBackendStepMetaData() .withName("prepare") - .withCode(new QCodeReference() - .withName(MockBackendStep.class.getName()) - .withCodeType(QCodeType.JAVA) - .withCodeUsage(QCodeUsage.BACKEND_STEP)) // todo - needed, or implied in this context? + .withCode(new QCodeReference(MockBackendStep.class)) .withInputData(new QFunctionInputMetaData() .withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON)) .withFieldList(List.of( @@ -354,6 +353,7 @@ public class SampleMetaDataProvider .addStep(new QFrontendStepMetaData() .withName("setup") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)) .withFormField(new QFieldMetaData("greetingPrefix", QFieldType.STRING)) .withFormField(new QFieldMetaData("greetingSuffix", QFieldType.STRING)) ) @@ -380,6 +380,8 @@ public class SampleMetaDataProvider .addStep(new QFrontendStepMetaData() .withName("results") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)) .withViewField(new QFieldMetaData("noOfPeopleGreeted", QFieldType.INTEGER)) .withViewField(new QFieldMetaData("outputMessage", QFieldType.STRING)) .withRecordListField(new QFieldMetaData("id", QFieldType.INTEGER)) @@ -442,10 +444,12 @@ public class SampleMetaDataProvider .withName(PROCESS_NAME_SLEEP_INTERACTIVE) .addStep(new QFrontendStepMetaData() .withName(SCREEN_0) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) .withFormField(new QFieldMetaData("outputMessage", QFieldType.STRING))) .addStep(SleeperStep.getMetaData()) .addStep(new QFrontendStepMetaData() .withName(SCREEN_1) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) .withFormField(new QFieldMetaData("outputMessage", QFieldType.STRING))); } From bc95899a50e898e9657291b9425e9167d32780f3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 29 Aug 2022 14:20:00 -0500 Subject: [PATCH 10/28] Increase coverage --- .../ClonePeopleTransformStepTest.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java new file mode 100644 index 00000000..1cddcaf1 --- /dev/null +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java @@ -0,0 +1,54 @@ +package com.kingsrook.sampleapp.processes.clonepeople; + + +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.sampleapp.SampleMetaDataProvider; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for ClonePeopleTransformStep + *******************************************************************************/ +class ClonePeopleTransformStepTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testProcessStep() throws QException + { + QInstance qInstance = SampleMetaDataProvider.defineInstance(); + + QueryInput queryInput = new QueryInput(qInstance); + queryInput.setTableName(SampleMetaDataProvider.TABLE_NAME_PERSON); + queryInput.setSession(new QSession()); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + ClonePeopleTransformStep clonePeopleTransformStep = new ClonePeopleTransformStep(); + + clonePeopleTransformStep.setInputRecordPage(queryOutput.getRecords()); + clonePeopleTransformStep.run(input, output); + + ArrayList processSummary = clonePeopleTransformStep.getProcessSummary(true); + + assertThat(processSummary) + .usingRecursiveFieldByFieldElementComparatorOnFields("status", "count") + .contains(new ProcessSummaryLine(Status.OK, 4, null)) + .contains(new ProcessSummaryLine(Status.ERROR, 1, null)); + } + +} \ No newline at end of file From 6142b8e703f9b90016e13b9d85310ff4de8eced5 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 29 Aug 2022 14:29:01 -0500 Subject: [PATCH 11/28] Update tests to run w/ h2 instead of mysql --- .../sampleapp/SampleMetaDataProviderTest.java | 36 ++++++++++++++++--- .../ClonePeopleTransformStepTest.java | 30 ++++++++++++++++ 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleMetaDataProviderTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleMetaDataProviderTest.java index 08b5c6ea..07cf4872 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleMetaDataProviderTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleMetaDataProviderTest.java @@ -45,11 +45,11 @@ import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledOnOs; -import org.junit.jupiter.api.condition.OS; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -58,9 +58,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* ** *******************************************************************************/ -@DisabledOnOs(OS.LINUX) // uses database; not available in CI at this time... -class SampleMetaDataProviderTest +public class SampleMetaDataProviderTest { + private static boolean originalUseMysqlValue = false; + + /******************************************************************************* ** @@ -73,11 +75,35 @@ class SampleMetaDataProviderTest + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeAll + static void beforeAll() throws Exception + { + originalUseMysqlValue = SampleMetaDataProvider.USE_MYSQL; + SampleMetaDataProvider.USE_MYSQL = false; + SampleMetaDataProviderTest.primeTestDatabase("prime-test-database.sql"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterAll + static void afterAll() + { + SampleMetaDataProvider.USE_MYSQL = originalUseMysqlValue; + } + + + /******************************************************************************* ** *******************************************************************************/ @SuppressWarnings("unchecked") - private void primeTestDatabase(String sqlFileName) throws Exception + public static void primeTestDatabase(String sqlFileName) throws Exception { ConnectionManager connectionManager = new ConnectionManager(); try(Connection connection = connectionManager.getConnection(SampleMetaDataProvider.defineRdbmsBackend())) diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java index 1cddcaf1..e27131a6 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java @@ -13,6 +13,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.sampleapp.SampleMetaDataProvider; +import com.kingsrook.sampleapp.SampleMetaDataProviderTest; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -22,6 +25,33 @@ import static org.assertj.core.api.Assertions.assertThat; *******************************************************************************/ class ClonePeopleTransformStepTest { + private static boolean originalUseMysqlValue = false; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeAll + static void beforeAll() throws Exception + { + originalUseMysqlValue = SampleMetaDataProvider.USE_MYSQL; + SampleMetaDataProvider.USE_MYSQL = false; + SampleMetaDataProviderTest.primeTestDatabase("prime-test-database.sql"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterAll + static void afterAll() + { + SampleMetaDataProvider.USE_MYSQL = originalUseMysqlValue; + } + + /******************************************************************************* ** From 48b8d295e3ebd62204449e186e0c3b1c922a1344 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 30 Aug 2022 11:46:46 -0500 Subject: [PATCH 12/28] initial checkin of quicksight dashboard widget POC, updated to remove hard coded credentials --- qqq-backend-core/pom.xml | 15 + .../dashboard/AbstractWidgetRenderer.java | 3 +- .../dashboard/QuickSightChartRenderer.java | 77 +++++ .../actions/dashboard/WidgetDataLoader.java | 9 +- .../model/dashboard/widgets/BarChart.java | 55 +--- .../core/model/dashboard/widgets/QWidget.java | 14 + .../dashboard/widgets/QuickSightChart.java | 139 ++++++++ .../core/model/metadata/QInstance.java | 14 +- .../metadata/dashboard/QWidgetMetaData.java | 6 +- .../dashboard/QWidgetMetaDataInterface.java | 42 +++ .../dashboard/QuickSightChartMetaData.java | 305 ++++++++++++++++++ .../PersonsByCreateDateBarChart.java | 3 +- .../rdbms/jdbc/ConnectionManagerTest.java | 15 +- .../javalin/PersonsByCreateDateBarChart.java | 3 +- qqq-sample-project/pom.xml | 6 +- .../sampleapp/SampleMetaDataProvider.java | 59 ++-- .../widgets/PersonsByCreateDateBarChart.java | 3 +- .../PersonsByCreateDateBarChartTest.java | 4 +- 18 files changed, 684 insertions(+), 88 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index 268945bc..8a952f04 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -36,11 +36,26 @@ + + + + software.amazon.awssdk + bom + 2.17.259 + pom + import + + + + + software.amazon.awssdk + quicksight + com.fasterxml.jackson.core jackson-databind diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java index bf8ca097..7919b928 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java @@ -3,6 +3,7 @@ package com.kingsrook.qqq.backend.core.actions.dashboard; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -15,6 +16,6 @@ public abstract class AbstractWidgetRenderer /******************************************************************************* ** *******************************************************************************/ - public abstract Object render(QInstance qInstance, QSession session) throws QException; + public abstract Object render(QInstance qInstance, QSession session, QWidgetMetaDataInterface qWidgetMetaData) throws QException; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java new file mode 100644 index 00000000..fd352e3e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java @@ -0,0 +1,77 @@ +package com.kingsrook.qqq.backend.core.actions.dashboard; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QuickSightChart; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QuickSightChartMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.services.quicksight.QuickSightClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.quicksight.model.GenerateEmbedUrlForRegisteredUserRequest; +import software.amazon.awssdk.services.quicksight.model.GenerateEmbedUrlForRegisteredUserResponse; +import software.amazon.awssdk.services.quicksight.model.RegisteredUserDashboardEmbeddingConfiguration; +import software.amazon.awssdk.services.quicksight.model.RegisteredUserEmbeddingExperienceConfiguration; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuickSightChartRenderer extends AbstractWidgetRenderer +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Object render(QInstance qInstance, QSession session, QWidgetMetaDataInterface metaData) throws QException + { + try + { + QuickSightChartMetaData quickSightMetaData = (QuickSightChartMetaData) metaData; + QuickSightClient quickSightClient = getQuickSightClient(quickSightMetaData); + + final RegisteredUserEmbeddingExperienceConfiguration experienceConfiguration = RegisteredUserEmbeddingExperienceConfiguration.builder() + .dashboard( + RegisteredUserDashboardEmbeddingConfiguration.builder() + .initialDashboardId(quickSightMetaData.getDashboardId()) + .build()) + .build(); + + final GenerateEmbedUrlForRegisteredUserRequest generateEmbedUrlForRegisteredUserRequest = GenerateEmbedUrlForRegisteredUserRequest.builder() + .awsAccountId(quickSightMetaData.getAccountId()) + .userArn(quickSightMetaData.getUserArn()) + .experienceConfiguration(experienceConfiguration) + .build(); + + final GenerateEmbedUrlForRegisteredUserResponse generateEmbedUrlForRegisteredUserResponse = quickSightClient.generateEmbedUrlForRegisteredUser(generateEmbedUrlForRegisteredUserRequest); + + String embedUrl = generateEmbedUrlForRegisteredUserResponse.embedUrl(); + return (new QuickSightChart(metaData.getName(), quickSightMetaData.getLabel(), embedUrl)); + } + catch(Exception e) + { + throw (new QException("Error rendering widget", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QuickSightClient getQuickSightClient(QuickSightChartMetaData metaData) + { + AwsBasicCredentials awsCredentials = AwsBasicCredentials.create(metaData.getAccessKey(), metaData.getSecretKey()); + + QuickSightClient amazonQuickSightClient = QuickSightClient.builder() + .credentialsProvider(StaticCredentialsProvider.create(awsCredentials)) + .region(Region.of(metaData.getRegion())) + .build(); + + return (amazonQuickSightClient); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java index a0e5d214..de346886 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java @@ -4,7 +4,7 @@ package com.kingsrook.qqq.backend.core.actions.dashboard; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -19,8 +19,9 @@ public class WidgetDataLoader *******************************************************************************/ public Object execute(QInstance qInstance, QSession session, String name) throws QException { - QWidgetMetaData widget = qInstance.getWidget(name); - AbstractWidgetRenderer widgetRenderer = QCodeLoader.getAdHoc(AbstractWidgetRenderer.class, widget.getCodeReference()); - return (widgetRenderer.render(qInstance, session)); + QWidgetMetaDataInterface widget = qInstance.getWidget(name); + AbstractWidgetRenderer widgetRenderer = QCodeLoader.getAdHoc(AbstractWidgetRenderer.class, widget.getCodeReference()); + Object w = widgetRenderer.render(qInstance, session, widget); + return (widgetRenderer.render(qInstance, session, widget)); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java index f71f0c66..2d8e8667 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java @@ -7,7 +7,7 @@ import java.util.List; /******************************************************************************* ** *******************************************************************************/ -public class BarChart +public class BarChart implements QWidget { /* @@ -19,9 +19,8 @@ public class BarChart }, */ - private String type = "barChart"; private String title; - private Data barChartData; + private Data barChartData; @@ -34,12 +33,23 @@ public class BarChart setBarChartData(new BarChart.Data() .withLabels(labels) .withDatasets(new BarChart.Data.DataSet() - .withLabel("Parcel Invoice Lines") + .withLabel(seriesLabel) .withData(data))); } + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public String getType() + { + return "barChart"; + } + + + /******************************************************************************* ** Getter for title ** @@ -108,46 +118,13 @@ public class BarChart - /******************************************************************************* - ** Getter for type - ** - *******************************************************************************/ - public String getType() - { - return type; - } - - - - /******************************************************************************* - ** Setter for type - ** - *******************************************************************************/ - public void setType(String type) - { - this.type = type; - } - - - /******************************************************************************* - ** Fluent setter for type - ** - *******************************************************************************/ - public BarChart withType(String type) - { - this.type = type; - return (this); - } - - - /******************************************************************************* ** *******************************************************************************/ public static class Data { private List labels; - private DataSet datasets; + private DataSet datasets; @@ -224,7 +201,7 @@ public class BarChart *******************************************************************************/ public static class DataSet { - private String label; + private String label; private List data; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java new file mode 100644 index 00000000..8c76fc52 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java @@ -0,0 +1,14 @@ +package com.kingsrook.qqq.backend.core.model.dashboard.widgets; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface QWidget +{ + /******************************************************************************* + ** Getter for type + *******************************************************************************/ + String getType(); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java new file mode 100644 index 00000000..9cccd508 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java @@ -0,0 +1,139 @@ +package com.kingsrook.qqq.backend.core.model.dashboard.widgets; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuickSightChart implements QWidget +{ + private String label; + private String name; + private String url; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QuickSightChart(String name, String label, String url) + { + this.url = url; + this.name = name; + this.label = label; + } + + + + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public String getType() + { + return "quickSightChart"; + } + + + + /******************************************************************************* + ** Getter for url + ** + *******************************************************************************/ + public String getUrl() + { + return url; + } + + + + /******************************************************************************* + ** Setter for url + ** + *******************************************************************************/ + public void setUrl(String url) + { + this.url = url; + } + + + + /******************************************************************************* + ** Fluent setter for url + ** + *******************************************************************************/ + public QuickSightChart withUrl(String url) + { + this.url = url; + return (this); + } + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Setter for name + ** + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public QuickSightChart withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + + return label; + } + + + + /******************************************************************************* + ** Setter for label + ** + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + ** + *******************************************************************************/ + public QuickSightChart withLabel(String label) + { + this.label = label; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 02718a01..d67e4ac6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.instances.QInstanceValidationKey; -import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; @@ -60,7 +60,7 @@ public class QInstance private Map processes = new LinkedHashMap<>(); private Map apps = new LinkedHashMap<>(); - private Map widgets = new LinkedHashMap<>(); + private Map widgets = new LinkedHashMap<>(); // todo - lock down the object (no more changes allowed) after it's been validated? @@ -455,7 +455,7 @@ public class QInstance ** Getter for widgets ** *******************************************************************************/ - public Map getWidgets() + public Map getWidgets() { return widgets; } @@ -466,7 +466,7 @@ public class QInstance ** Setter for widgets ** *******************************************************************************/ - public void setWidgets(Map widgets) + public void setWidgets(Map widgets) { this.widgets = widgets; } @@ -475,7 +475,7 @@ public class QInstance /******************************************************************************* ** *******************************************************************************/ - public void addWidget(QWidgetMetaData widget) + public void addWidget(QWidgetMetaDataInterface widget) { this.addWidget(widget.getName(), widget); } @@ -485,7 +485,7 @@ public class QInstance /******************************************************************************* ** *******************************************************************************/ - public void addWidget(String name, QWidgetMetaData widget) + public void addWidget(String name, QWidgetMetaDataInterface widget) { if(this.widgets.containsKey(name)) { @@ -499,7 +499,7 @@ public class QInstance /******************************************************************************* ** *******************************************************************************/ - public QWidgetMetaData getWidget(String name) + public QWidgetMetaDataInterface getWidget(String name) { return (this.widgets.get(name)); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java index 0efb0dfe..2a9594b7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java @@ -7,10 +7,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; /******************************************************************************* ** *******************************************************************************/ -public class QWidgetMetaData +public class QWidgetMetaData implements QWidgetMetaDataInterface { - private String name; - private QCodeReference codeReference; + protected String name; + protected QCodeReference codeReference; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java new file mode 100644 index 00000000..771c8d33 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java @@ -0,0 +1,42 @@ +package com.kingsrook.qqq.backend.core.model.metadata.dashboard; + + +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface QWidgetMetaDataInterface +{ + /******************************************************************************* + ** Getter for name + *******************************************************************************/ + String getName(); + + /******************************************************************************* + ** Setter for name + *******************************************************************************/ + void setName(String name); + + /******************************************************************************* + ** Fluent setter for name + *******************************************************************************/ + QWidgetMetaDataInterface withName(String name); + + /******************************************************************************* + ** Getter for codeReference + *******************************************************************************/ + QCodeReference getCodeReference(); + + /******************************************************************************* + ** Setter for codeReference + *******************************************************************************/ + void setCodeReference(QCodeReference codeReference); + + /******************************************************************************* + ** Fluent setter for codeReference + *******************************************************************************/ + QWidgetMetaDataInterface withCodeReference(QCodeReference codeReference); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java new file mode 100644 index 00000000..114804a4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java @@ -0,0 +1,305 @@ +package com.kingsrook.qqq.backend.core.model.metadata.dashboard; + + +import java.util.Collection; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class QuickSightChartMetaData extends QWidgetMetaData implements QWidgetMetaDataInterface +{ + private String label; + private String accessKey; + private String secretKey; + private String dashboardId; + private String accountId; + private String userArn; + private String region; + private Collection allowedDomains; + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public QuickSightChartMetaData withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for accessKey + ** + *******************************************************************************/ + public String getAccessKey() + { + return accessKey; + } + + + + /******************************************************************************* + ** Setter for accessKey + ** + *******************************************************************************/ + public void setAccessKey(String accessKey) + { + this.accessKey = accessKey; + } + + + + /******************************************************************************* + ** Fluent setter for accessKey + ** + *******************************************************************************/ + public QuickSightChartMetaData withAccessKey(String accessKey) + { + this.accessKey = accessKey; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + + return label; + } + + + + /******************************************************************************* + ** Setter for label + ** + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + ** + *******************************************************************************/ + public QuickSightChartMetaData withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for secretKey + ** + *******************************************************************************/ + public String getSecretKey() + { + return secretKey; + } + + + + /******************************************************************************* + ** Setter for secretKey + ** + *******************************************************************************/ + public void setSecretKey(String secretKey) + { + this.secretKey = secretKey; + } + + + + /******************************************************************************* + ** Fluent setter for secretKey + ** + *******************************************************************************/ + public QuickSightChartMetaData withSecretKey(String secretKey) + { + this.secretKey = secretKey; + return (this); + } + + + + /******************************************************************************* + ** Getter for dashboardId + ** + *******************************************************************************/ + public String getDashboardId() + { + return dashboardId; + } + + + + /******************************************************************************* + ** Setter for dashboardId + ** + *******************************************************************************/ + public void setDashboardId(String dashboardId) + { + this.dashboardId = dashboardId; + } + + + + /******************************************************************************* + ** Fluent setter for dashboardId + ** + *******************************************************************************/ + public QuickSightChartMetaData withDashboardId(String dashboardId) + { + this.dashboardId = dashboardId; + return (this); + } + + + + /******************************************************************************* + ** Getter for accountId + ** + *******************************************************************************/ + public String getAccountId() + { + return accountId; + } + + + + /******************************************************************************* + ** Setter for accountId + ** + *******************************************************************************/ + public void setAccountId(String accountId) + { + this.accountId = accountId; + } + + + + /******************************************************************************* + ** Fluent setter for accountId + ** + *******************************************************************************/ + public QuickSightChartMetaData withAccountId(String accountId) + { + this.accountId = accountId; + return this; + } + + + + /******************************************************************************* + ** Getter for userArn + ** + *******************************************************************************/ + public String getUserArn() + { + return userArn; + } + + + + /******************************************************************************* + ** Setter for userArn + ** + *******************************************************************************/ + public void setUserArn(String userArn) + { + this.userArn = userArn; + } + + + + /******************************************************************************* + ** Fluent setter for userArn + ** + *******************************************************************************/ + public QuickSightChartMetaData withUserArn(String userArn) + { + this.userArn = userArn; + return this; + } + + + + /******************************************************************************* + ** Getter for region + ** + *******************************************************************************/ + public String getRegion() + { + return region; + } + + + + /******************************************************************************* + ** Setter for region + ** + *******************************************************************************/ + public void setRegion(String region) + { + this.region = region; + } + + + + /******************************************************************************* + ** Fluent setter for region + ** + *******************************************************************************/ + public QuickSightChartMetaData withRegion(String region) + { + this.region = region; + return this; + } + + + + /******************************************************************************* + ** Getter for allowedDomains + ** + *******************************************************************************/ + public Collection getAllowedDomains() + { + return allowedDomains; + } + + + + /******************************************************************************* + ** Setter for allowedDomains + ** + *******************************************************************************/ + public void setAllowedDomains(Collection allowedDomains) + { + this.allowedDomains = allowedDomains; + } + + + + /******************************************************************************* + ** Fluent setter for allowedDomains + ** + *******************************************************************************/ + public QuickSightChartMetaData withAllowedDomains(Collection allowedDomains) + { + this.allowedDomains = allowedDomains; + return this; + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/PersonsByCreateDateBarChart.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/PersonsByCreateDateBarChart.java index da2c1ff7..06997d0a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/PersonsByCreateDateBarChart.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/PersonsByCreateDateBarChart.java @@ -6,6 +6,7 @@ import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.BarChart; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -18,7 +19,7 @@ public class PersonsByCreateDateBarChart extends AbstractWidgetRenderer ** *******************************************************************************/ @Override - public Object render(QInstance qInstance, QSession session) throws QException + public Object render(QInstance qInstance, QSession session, QWidgetMetaDataInterface metaData) throws QException { try { diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java index a51a041b..c23cd928 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java @@ -25,7 +25,9 @@ package com.kingsrook.qqq.backend.module.rdbms.jdbc; import java.sql.Connection; import java.sql.SQLException; import java.util.Collections; +import java.util.Objects; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import io.github.cdimascio.dotenv.Dotenv; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -101,13 +103,14 @@ class ConnectionManagerTest private RDBMSBackendMetaData getAuroraBacked() { + Dotenv dotenv = Dotenv.configure().load(); return new RDBMSBackendMetaData() .withName("aurora-test") - .withVendor("aurora") - .withHostName("nf-one-development-aurora.cwuhqcx1inwx.us-east-2.rds.amazonaws.com") - .withPort(3306) - .withDatabaseName("nutrifresh_one") - .withUsername("nf_admin") - .withPassword("%!2rwcH+fb#WgPg"); + .withVendor(dotenv.get("RDBMS_VENDOR")) + .withHostName(dotenv.get("RDBMS_HOSTNAME")) + .withPort(Integer.valueOf(Objects.requireNonNull(dotenv.get("RDBMS_PORT")))) + .withDatabaseName(dotenv.get("RDBMS_DATABASE_NAME")) + .withUsername(dotenv.get("RDBMS_USERNAME")) + .withPassword(dotenv.get("RDBMS_PASSWORD")); } } diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java index 506b099e..5afbf0d6 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java @@ -7,6 +7,7 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.AbstractWidgetRenderer; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.BarChart; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -19,7 +20,7 @@ public class PersonsByCreateDateBarChart extends AbstractWidgetRenderer ** *******************************************************************************/ @Override - public Object render(QInstance qInstance, QSession session) throws QException + public Object render(QInstance qInstance, QSession session, QWidgetMetaDataInterface metaData) throws QException { try { diff --git a/qqq-sample-project/pom.xml b/qqq-sample-project/pom.xml index 23f84047..e1d80a71 100644 --- a/qqq-sample-project/pom.xml +++ b/qqq-sample-project/pom.xml @@ -128,9 +128,9 @@ 4.10.0 /src/main/resources/liquibase/liquibase.properties - ${env.LB_DB_URL} - ${env.LB_DB_USERNAME} - ${env.LB_DB_PASSWORD} + ${env.RDBMS_URL} + ${env.RDBMS_USERNAME} + ${env.RDBMS_PASSWORD} ${env.LB_CONTEXTS} diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index 76086502..2330845c 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -23,6 +23,9 @@ package com.kingsrook.sampleapp; import java.util.List; +import java.util.Objects; +import com.amazonaws.regions.Regions; +import com.kingsrook.qqq.backend.core.actions.dashboard.QuickSightChartRenderer; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QValueException; @@ -35,6 +38,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QuickSightChartMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; @@ -68,26 +73,22 @@ public class SampleMetaDataProvider { public static boolean USE_MYSQL = true; - public static final String RDBMS_BACKEND_NAME = "rdbms"; + public static final String RDBMS_BACKEND_NAME = "rdbms"; public static final String FILESYSTEM_BACKEND_NAME = "filesystem"; - public static final String AUTH0_AUTHENTICATION_MODULE_NAME = "auth0"; - // public static final String AUTH0_BASE_URL = "https://kingsrook.us.auth0.com/"; - public static final String AUTH0_BASE_URL = "https://nutrifresh-one-development.us.auth0.com/"; - - public static final String APP_NAME_GREETINGS = "greetingsApp"; - public static final String APP_NAME_PEOPLE = "peopleApp"; + public static final String APP_NAME_GREETINGS = "greetingsApp"; + public static final String APP_NAME_PEOPLE = "peopleApp"; public static final String APP_NAME_MISCELLANEOUS = "miscellaneous"; - public static final String PROCESS_NAME_GREET = "greet"; + public static final String PROCESS_NAME_GREET = "greet"; public static final String PROCESS_NAME_GREET_INTERACTIVE = "greetInteractive"; - public static final String PROCESS_NAME_SIMPLE_SLEEP = "simpleSleep"; - public static final String PROCESS_NAME_SIMPLE_THROW = "simpleThrow"; + public static final String PROCESS_NAME_SIMPLE_SLEEP = "simpleSleep"; + public static final String PROCESS_NAME_SIMPLE_THROW = "simpleThrow"; public static final String PROCESS_NAME_SLEEP_INTERACTIVE = "sleepInteractive"; - public static final String TABLE_NAME_PERSON = "person"; + public static final String TABLE_NAME_PERSON = "person"; public static final String TABLE_NAME_CARRIER = "carrier"; - public static final String TABLE_NAME_CITY = "city"; + public static final String TABLE_NAME_CITY = "city"; public static final String STEP_NAME_SLEEPER = "sleeper"; public static final String STEP_NAME_THROWER = "thrower"; @@ -133,6 +134,20 @@ public class SampleMetaDataProvider qInstance.addWidget(new QWidgetMetaData() .withName(PersonsByCreateDateBarChart.class.getSimpleName()) .withCodeReference(new QCodeReference(PersonsByCreateDateBarChart.class, null))); + + Dotenv dotenv = Dotenv.configure().load(); + QWidgetMetaDataInterface quickSightChartMetaData = new QuickSightChartMetaData() + .withAccountId(dotenv.get("QUICKSIGHT_ACCCOUNT_ID")) + .withAccessKey(dotenv.get("QUICKSIGHT_ACCESS_KEY")) + .withSecretKey(dotenv.get("QUICKSIGHT_SECRET_KEY")) + .withUserArn(dotenv.get("QUICKSIGHT_USER_ARN")) + .withDashboardId("9e452e78-8509-4c81-bb7f-967abfc356da") + .withRegion(Regions.US_EAST_2.getName()) + .withName(QuickSightChartRenderer.class.getSimpleName()) + .withLabel("Example Quicksight Chart") + .withCodeReference(new QCodeReference(QuickSightChartRenderer.class, null)); + + qInstance.addWidget(quickSightChartMetaData); } @@ -153,7 +168,11 @@ public class SampleMetaDataProvider .withIcon(new QIcon().withName("location_city"))) .withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE)) .withIcon(new QIcon().withName("waving_hand")) - .withWidgets(List.of(PersonsByCreateDateBarChart.class.getSimpleName())) + .withWidgets(List.of + ( + PersonsByCreateDateBarChart.class.getSimpleName(), + QuickSightChartRenderer.class.getSimpleName() + )) ); qInstance.addApp(new QAppMetaData() @@ -194,11 +213,11 @@ public class SampleMetaDataProvider Dotenv dotenv = Dotenv.configure().load(); return new RDBMSBackendMetaData() .withName(RDBMS_BACKEND_NAME) - .withVendor("mysql") - .withHostName("127.0.0.1") - .withPort(3306) - .withDatabaseName("qqq") - .withUsername("root") + .withVendor(dotenv.get("RDBMS_VENDOR")) + .withHostName(dotenv.get("RDBMS_HOSTNAME")) + .withPort(Integer.valueOf(Objects.requireNonNull(dotenv.get("RDBMS_PORT")))) + .withDatabaseName(dotenv.get("RDBMS_DATABASE_NAME")) + .withUsername(dotenv.get("RDBMS_USERNAME")) .withPassword(dotenv.get("RDBMS_PASSWORD")); } else @@ -249,8 +268,8 @@ public class SampleMetaDataProvider .withBackendName("company_code")); table.addField(new QFieldMetaData("service_level", QFieldType.STRING) // todo PVS - .withLabel("Service Level") - .withIsRequired(true)); + .withLabel("Service Level") + .withIsRequired(true)); table.addSection(new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, List.of("id", "name"))); table.addSection(new QFieldSection("basicInfo", "Basic Info", new QIcon("dataset"), Tier.T2, List.of("company_code", "service_level"))); diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java index 29ec412b..490ac39f 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java @@ -7,6 +7,7 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.AbstractWidgetRenderer; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.BarChart; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -19,7 +20,7 @@ public class PersonsByCreateDateBarChart extends AbstractWidgetRenderer ** *******************************************************************************/ @Override - public Object render(QInstance qInstance, QSession session) throws QException + public Object render(QInstance qInstance, QSession session, QWidgetMetaDataInterface metaData) throws QException { try { diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChartTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChartTest.java index 09d8a3ae..b74f98ba 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChartTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChartTest.java @@ -23,7 +23,7 @@ class PersonsByCreateDateBarChartTest @Test void test() throws QException { - Object widgetData = new PersonsByCreateDateBarChart().render(SampleMetaDataProvider.defineInstance(), new QSession()); + Object widgetData = new PersonsByCreateDateBarChart().render(SampleMetaDataProvider.defineInstance(), new QSession(), null); assertThat(widgetData).isInstanceOf(BarChart.class); BarChart barChart = (BarChart) widgetData; assertEquals("barChart", barChart.getType()); @@ -31,4 +31,4 @@ class PersonsByCreateDateBarChartTest assertNotNull(barChart.getBarChartData()); } -} \ No newline at end of file +} From 4316b4791605bc3c171c874abc64325ec417b18e Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 30 Aug 2022 12:13:29 -0500 Subject: [PATCH 13/28] fixed incorrect import order --- .../backend/core/actions/dashboard/QuickSightChartRenderer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java index fd352e3e..74687c75 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java @@ -9,8 +9,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QuickSightChartMe import com.kingsrook.qqq.backend.core.model.session.QSession; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.services.quicksight.QuickSightClient; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.quicksight.QuickSightClient; import software.amazon.awssdk.services.quicksight.model.GenerateEmbedUrlForRegisteredUserRequest; import software.amazon.awssdk.services.quicksight.model.GenerateEmbedUrlForRegisteredUserResponse; import software.amazon.awssdk.services.quicksight.model.RegisteredUserDashboardEmbeddingConfiguration; From 4c2ebf8a94bcb864586da37e5ab1084d674b4cd8 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 30 Aug 2022 12:17:37 -0500 Subject: [PATCH 14/28] PRDONE-94: updated to ignore missing .env file --- .../qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java | 4 +++- .../java/com/kingsrook/sampleapp/SampleMetaDataProvider.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java index c23cd928..715e4358 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java @@ -103,7 +103,9 @@ class ConnectionManagerTest private RDBMSBackendMetaData getAuroraBacked() { - Dotenv dotenv = Dotenv.configure().load(); + Dotenv dotenv = Dotenv.configure() + .ignoreIfMissing() + .load(); return new RDBMSBackendMetaData() .withName("aurora-test") .withVendor(dotenv.get("RDBMS_VENDOR")) diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index 2330845c..fd48661d 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -135,7 +135,9 @@ public class SampleMetaDataProvider .withName(PersonsByCreateDateBarChart.class.getSimpleName()) .withCodeReference(new QCodeReference(PersonsByCreateDateBarChart.class, null))); - Dotenv dotenv = Dotenv.configure().load(); + Dotenv dotenv = Dotenv.configure() + .ignoreIfMissing() + .load(); QWidgetMetaDataInterface quickSightChartMetaData = new QuickSightChartMetaData() .withAccountId(dotenv.get("QUICKSIGHT_ACCCOUNT_ID")) .withAccessKey(dotenv.get("QUICKSIGHT_ACCESS_KEY")) From dcea96579c6293a77b43a0c0536102113b29e233 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 30 Aug 2022 12:28:40 -0500 Subject: [PATCH 15/28] PRDONE-94: updated to set dotenv variables as system properties --- .../rdbms/jdbc/ConnectionManagerTest.java | 13 +++++----- .../sampleapp/SampleMetaDataProvider.java | 26 +++++++++++-------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java index 715e4358..8a9ce4a9 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java @@ -104,15 +104,16 @@ class ConnectionManagerTest private RDBMSBackendMetaData getAuroraBacked() { Dotenv dotenv = Dotenv.configure() + .systemProperties() .ignoreIfMissing() .load(); return new RDBMSBackendMetaData() .withName("aurora-test") - .withVendor(dotenv.get("RDBMS_VENDOR")) - .withHostName(dotenv.get("RDBMS_HOSTNAME")) - .withPort(Integer.valueOf(Objects.requireNonNull(dotenv.get("RDBMS_PORT")))) - .withDatabaseName(dotenv.get("RDBMS_DATABASE_NAME")) - .withUsername(dotenv.get("RDBMS_USERNAME")) - .withPassword(dotenv.get("RDBMS_PASSWORD")); + .withVendor(System.getProperty("RDBMS_VENDOR")) + .withHostName(System.getProperty("RDBMS_HOSTNAME")) + .withPort(Integer.valueOf(Objects.requireNonNull(System.getProperty("RDBMS_PORT")))) + .withDatabaseName(System.getProperty("RDBMS_DATABASE_NAME")) + .withUsername(System.getProperty("RDBMS_USERNAME")) + .withPassword(System.getProperty("RDBMS_PASSWORD")); } } diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index fd48661d..d024c0a8 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -137,12 +137,13 @@ public class SampleMetaDataProvider Dotenv dotenv = Dotenv.configure() .ignoreIfMissing() + .systemProperties() .load(); QWidgetMetaDataInterface quickSightChartMetaData = new QuickSightChartMetaData() - .withAccountId(dotenv.get("QUICKSIGHT_ACCCOUNT_ID")) - .withAccessKey(dotenv.get("QUICKSIGHT_ACCESS_KEY")) - .withSecretKey(dotenv.get("QUICKSIGHT_SECRET_KEY")) - .withUserArn(dotenv.get("QUICKSIGHT_USER_ARN")) + .withAccountId(System.getProperty("QUICKSIGHT_ACCCOUNT_ID")) + .withAccessKey(System.getProperty("QUICKSIGHT_ACCESS_KEY")) + .withSecretKey(System.getProperty("QUICKSIGHT_SECRET_KEY")) + .withUserArn(System.getProperty("QUICKSIGHT_USER_ARN")) .withDashboardId("9e452e78-8509-4c81-bb7f-967abfc356da") .withRegion(Regions.US_EAST_2.getName()) .withName(QuickSightChartRenderer.class.getSimpleName()) @@ -212,15 +213,18 @@ public class SampleMetaDataProvider { if(USE_MYSQL) { - Dotenv dotenv = Dotenv.configure().load(); + Dotenv dotenv = Dotenv.configure() + .ignoreIfMissing() + .systemProperties() + .load(); return new RDBMSBackendMetaData() .withName(RDBMS_BACKEND_NAME) - .withVendor(dotenv.get("RDBMS_VENDOR")) - .withHostName(dotenv.get("RDBMS_HOSTNAME")) - .withPort(Integer.valueOf(Objects.requireNonNull(dotenv.get("RDBMS_PORT")))) - .withDatabaseName(dotenv.get("RDBMS_DATABASE_NAME")) - .withUsername(dotenv.get("RDBMS_USERNAME")) - .withPassword(dotenv.get("RDBMS_PASSWORD")); + .withVendor(System.getProperty("RDBMS_VENDOR")) + .withHostName(System.getProperty("RDBMS_HOSTNAME")) + .withPort(Integer.valueOf(Objects.requireNonNull(System.getProperty("RDBMS_PORT")))) + .withDatabaseName(System.getProperty("RDBMS_DATABASE_NAME")) + .withUsername(System.getProperty("RDBMS_USERNAME")) + .withPassword(System.getProperty("RDBMS_PASSWORD")); } else { From 4bf1fe863886cfd7521518d9798f3da9fe513b18 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 30 Aug 2022 13:33:13 -0500 Subject: [PATCH 16/28] PRDONE-94: updated to look for dotenv properties and fall back to environment vars --- .../rdbms/jdbc/ConnectionManagerTest.java | 24 ++++--- .../sampleapp/SampleMetaDataProvider.java | 64 ++++++++++--------- 2 files changed, 47 insertions(+), 41 deletions(-) diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java index 8a9ce4a9..0c063dd1 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java @@ -103,17 +103,21 @@ class ConnectionManagerTest private RDBMSBackendMetaData getAuroraBacked() { - Dotenv dotenv = Dotenv.configure() - .systemProperties() - .ignoreIfMissing() - .load(); + Dotenv dotenv = Dotenv.configure().ignoreIfMissing().load(); + String vendor = (dotenv.get("RDBMS_VENDOR") != null) ? dotenv.get("RDBMS_VENDOR") : System.getenv("RDBMS_VENDOR"); + String hostname = (dotenv.get("RDBMS_HOSTNAME") != null) ? dotenv.get("RDBMS_HOSTNAME") : System.getenv("RDBMS_HOSTNAME"); + Integer port = (dotenv.get("RDBMS_PORT") != null) ? Integer.valueOf(Objects.requireNonNull(dotenv.get("RDBMS_PORT"))) : Integer.valueOf(System.getenv("RDBMS_PORT")); + String databaseName = (dotenv.get("RDBMS_DATABASE_NAME") != null) ? dotenv.get("RDBMS_DATABASE_NAME") : System.getenv("RDBMS_DATABASE_NAME"); + String userName = (dotenv.get("RDBMS_USERNAME") != null) ? dotenv.get("RDBMS_USERNAME") : System.getenv("RDBMS_USERNAME"); + String password = (dotenv.get("RDBMS_PASSWORD") != null) ? dotenv.get("RDBMS_PASSWORD") : System.getenv("RDBMS_PASSWORD"); + return new RDBMSBackendMetaData() .withName("aurora-test") - .withVendor(System.getProperty("RDBMS_VENDOR")) - .withHostName(System.getProperty("RDBMS_HOSTNAME")) - .withPort(Integer.valueOf(Objects.requireNonNull(System.getProperty("RDBMS_PORT")))) - .withDatabaseName(System.getProperty("RDBMS_DATABASE_NAME")) - .withUsername(System.getProperty("RDBMS_USERNAME")) - .withPassword(System.getProperty("RDBMS_PASSWORD")); + .withVendor(vendor) + .withHostName(hostname) + .withPort(port) + .withDatabaseName(databaseName) + .withUsername(userName) + .withPassword(password); } } diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index d024c0a8..db74876f 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -73,22 +73,22 @@ public class SampleMetaDataProvider { public static boolean USE_MYSQL = true; - public static final String RDBMS_BACKEND_NAME = "rdbms"; + public static final String RDBMS_BACKEND_NAME = "rdbms"; public static final String FILESYSTEM_BACKEND_NAME = "filesystem"; - public static final String APP_NAME_GREETINGS = "greetingsApp"; - public static final String APP_NAME_PEOPLE = "peopleApp"; + public static final String APP_NAME_GREETINGS = "greetingsApp"; + public static final String APP_NAME_PEOPLE = "peopleApp"; public static final String APP_NAME_MISCELLANEOUS = "miscellaneous"; - public static final String PROCESS_NAME_GREET = "greet"; + public static final String PROCESS_NAME_GREET = "greet"; public static final String PROCESS_NAME_GREET_INTERACTIVE = "greetInteractive"; - public static final String PROCESS_NAME_SIMPLE_SLEEP = "simpleSleep"; - public static final String PROCESS_NAME_SIMPLE_THROW = "simpleThrow"; + public static final String PROCESS_NAME_SIMPLE_SLEEP = "simpleSleep"; + public static final String PROCESS_NAME_SIMPLE_THROW = "simpleThrow"; public static final String PROCESS_NAME_SLEEP_INTERACTIVE = "sleepInteractive"; - public static final String TABLE_NAME_PERSON = "person"; + public static final String TABLE_NAME_PERSON = "person"; public static final String TABLE_NAME_CARRIER = "carrier"; - public static final String TABLE_NAME_CITY = "city"; + public static final String TABLE_NAME_CITY = "city"; public static final String STEP_NAME_SLEEPER = "sleeper"; public static final String STEP_NAME_THROWER = "thrower"; @@ -135,15 +135,17 @@ public class SampleMetaDataProvider .withName(PersonsByCreateDateBarChart.class.getSimpleName()) .withCodeReference(new QCodeReference(PersonsByCreateDateBarChart.class, null))); - Dotenv dotenv = Dotenv.configure() - .ignoreIfMissing() - .systemProperties() - .load(); + Dotenv dotenv = Dotenv.configure().ignoreIfMissing().load(); + String accountId = (dotenv.get("QUICKSIGHT_ACCOUNT_ID") != null) ? dotenv.get("QUICKSIGHT_ACCOUNT_ID") : System.getenv("QUICKSIGHT_ACCOUNT_ID"); + String accessKey = (dotenv.get("QUICKSIGHT_ACCESS_KEY") != null) ? dotenv.get("QUICKSIGHT_ACCESS_KEY") : System.getenv("QUICKSIGHT_ACCESS_KEY"); + String secretKey = (dotenv.get("QUICKSIGHT_SECRET_KEY") != null) ? dotenv.get("QUICKSIGHT_SECRET_KEY") : System.getenv("QUICKSIGHT_SECRET_KEY"); + String userArn = (dotenv.get("QUICKSIGHT_USER_ARN") != null) ? dotenv.get("QUICKSIGHT_USER_ARN") : System.getenv("QUICKSIGHT_USER_ARN"); + QWidgetMetaDataInterface quickSightChartMetaData = new QuickSightChartMetaData() - .withAccountId(System.getProperty("QUICKSIGHT_ACCCOUNT_ID")) - .withAccessKey(System.getProperty("QUICKSIGHT_ACCESS_KEY")) - .withSecretKey(System.getProperty("QUICKSIGHT_SECRET_KEY")) - .withUserArn(System.getProperty("QUICKSIGHT_USER_ARN")) + .withAccountId(accountId) + .withAccessKey(accessKey) + .withSecretKey(secretKey) + .withUserArn(userArn) .withDashboardId("9e452e78-8509-4c81-bb7f-967abfc356da") .withRegion(Regions.US_EAST_2.getName()) .withName(QuickSightChartRenderer.class.getSimpleName()) @@ -171,11 +173,7 @@ public class SampleMetaDataProvider .withIcon(new QIcon().withName("location_city"))) .withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE)) .withIcon(new QIcon().withName("waving_hand")) - .withWidgets(List.of - ( - PersonsByCreateDateBarChart.class.getSimpleName(), - QuickSightChartRenderer.class.getSimpleName() - )) + .withWidgets(List.of(PersonsByCreateDateBarChart.class.getSimpleName(), QuickSightChartRenderer.class.getSimpleName())) ); qInstance.addApp(new QAppMetaData() @@ -213,18 +211,22 @@ public class SampleMetaDataProvider { if(USE_MYSQL) { - Dotenv dotenv = Dotenv.configure() - .ignoreIfMissing() - .systemProperties() - .load(); + Dotenv dotenv = Dotenv.configure().ignoreIfMissing().load(); + String vendor = (dotenv.get("RDBMS_VENDOR") != null) ? dotenv.get("RDBMS_VENDOR") : System.getenv("RDBMS_VENDOR"); + String hostname = (dotenv.get("RDBMS_HOSTNAME") != null) ? dotenv.get("RDBMS_HOSTNAME") : System.getenv("RDBMS_HOSTNAME"); + Integer port = (dotenv.get("RDBMS_PORT") != null) ? Integer.valueOf(Objects.requireNonNull(dotenv.get("RDBMS_PORT"))) : Integer.valueOf(System.getenv("RDBMS_PORT")); + String databaseName = (dotenv.get("RDBMS_DATABASE_NAME") != null) ? dotenv.get("RDBMS_DATABASE_NAME") : System.getenv("RDBMS_DATABASE_NAME"); + String userName = (dotenv.get("RDBMS_USERNAME") != null) ? dotenv.get("RDBMS_USERNAME") : System.getenv("RDBMS_USERNAME"); + String password = (dotenv.get("RDBMS_PASSWORD") != null) ? dotenv.get("RDBMS_PASSWORD") : System.getenv("RDBMS_PASSWORD"); + return new RDBMSBackendMetaData() .withName(RDBMS_BACKEND_NAME) - .withVendor(System.getProperty("RDBMS_VENDOR")) - .withHostName(System.getProperty("RDBMS_HOSTNAME")) - .withPort(Integer.valueOf(Objects.requireNonNull(System.getProperty("RDBMS_PORT")))) - .withDatabaseName(System.getProperty("RDBMS_DATABASE_NAME")) - .withUsername(System.getProperty("RDBMS_USERNAME")) - .withPassword(System.getProperty("RDBMS_PASSWORD")); + .withVendor(vendor) + .withHostName(hostname) + .withPort(port) + .withDatabaseName(databaseName) + .withUsername(userName) + .withPassword(password); } else { From 69b9ed5b193b02a0a5a8ef4df0930bc0545d6b0b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 30 Aug 2022 13:44:34 -0500 Subject: [PATCH 17/28] QQQ-37 adding pre & post run to ETL transform & load; minor QOL --- .../core/actions/reporting/RecordPipe.java | 23 +++++++++++--- .../values/QPossibleValueTranslator.java | 2 +- .../core/instances/QInstanceEnricher.java | 31 ++++++++++++------- .../model/metadata/fields/QFieldMetaData.java | 11 +++++++ .../metadata/processes/QProcessMetaData.java | 11 +++++++ .../AbstractLoadStep.java | 27 ++++++++++++++++ .../AbstractTransformStep.java | 31 ++++++++++++++++++- .../BaseStreamedETLStep.java | 24 ++++++++++++++ .../StreamedETLExecuteStep.java | 23 +++++++++----- .../StreamedETLPreviewStep.java | 4 +++ .../StreamedETLValidateStep.java | 7 ++++- .../StreamedETLWithFrontendProcess.java | 8 ++++- .../javalin/QJavalinProcessHandler.java | 20 +++++++----- .../sampleapp/SampleMetaDataProvider.java | 1 + 14 files changed, 189 insertions(+), 34 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java index 4aee9e2d..87b348b7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java @@ -41,6 +41,9 @@ public class RecordPipe { private static final Logger LOG = LogManager.getLogger(RecordPipe.class); + private static final long BLOCKING_SLEEP_MILLIS = 100; + private static final long MAX_SLEEP_LOOP_MILLIS = 300_000; // 5 minutes + private ArrayBlockingQueue queue = new ArrayBlockingQueue<>(1_000); private boolean isTerminated = false; @@ -53,6 +56,7 @@ public class RecordPipe private List singleRecordListForPostRecordActions = new ArrayList<>(); + /******************************************************************************* ** Turn off the pipe. Stop accepting new records (just ignore them in the add ** method). Clear the existing queue. Don't return any more records. Note that @@ -103,11 +107,22 @@ public class RecordPipe { boolean offerResult = queue.offer(record); - while(!offerResult && !isTerminated) + if(!offerResult && !isTerminated) { - LOG.debug("Record pipe.add failed (due to full pipe). Blocking."); - SleepUtils.sleep(100, TimeUnit.MILLISECONDS); - offerResult = queue.offer(record); + long sleepLoopStartTime = System.currentTimeMillis(); + long now = System.currentTimeMillis(); + while(!offerResult && !isTerminated) + { + if(now - sleepLoopStartTime > MAX_SLEEP_LOOP_MILLIS) + { + LOG.warn("Giving up adding record to pipe, due to pipe being full for more than {} millis", MAX_SLEEP_LOOP_MILLIS); + throw (new IllegalStateException("Giving up adding record to pipe, due to pipe staying full too long.")); + } + LOG.debug("Record pipe.add failed (due to full pipe). Blocking."); + SleepUtils.sleep(BLOCKING_SLEEP_MILLIS, TimeUnit.MILLISECONDS); + offerResult = queue.offer(record); + now = System.currentTimeMillis(); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java index 5c50cf2c..582dc7d4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java @@ -89,7 +89,7 @@ public class QPossibleValueTranslator *******************************************************************************/ public void translatePossibleValuesInRecords(QTableMetaData table, List records) { - if(records == null) + if(records == null || table == null) { return; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 19beae30..765d26b8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -307,8 +307,8 @@ public class QInstanceEnricher .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withIsRequired(true)) .withComponent(new QFrontendComponentMetaData() .withType(QComponentType.HELP_TEXT) - // .withValue("text", "Upload a CSV or XLSX file with the following columns: " + fieldsForHelpText)); - .withValue("text", "Upload a CSV file with the following columns: " + fieldsForHelpText)); + .withValue("text", "Upload a CSV file with the following columns: " + fieldsForHelpText)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)); QBackendStepMetaData receiveFileStep = new QBackendStepMetaData() .withName("receiveFile") @@ -322,7 +322,9 @@ public class QInstanceEnricher .withComponent(new QFrontendComponentMetaData() .withType(QComponentType.HELP_TEXT) .withValue("text", "The records below were parsed from your file, and will be inserted if you click Submit.")) - .withViewField(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER).withLabel("# of file rows")); + .withViewField(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER).withLabel("# of file rows")) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); QBackendStepMetaData storeStep = new QBackendStepMetaData() .withName("storeRecords") @@ -336,7 +338,9 @@ public class QInstanceEnricher .withComponent(new QFrontendComponentMetaData() .withType(QComponentType.HELP_TEXT) .withValue("text", "The records below have been inserted.")) - .withViewField(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER).withLabel("# of file rows")); + .withViewField(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER).withLabel("# of file rows")) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); qInstance.addProcess( new QProcessMetaData() @@ -375,9 +379,7 @@ public class QInstanceEnricher The values you supply here will be updated in all of the records you are bulk editing. You can clear out the value in a field by flipping the switch on for that field and leaving the input field blank. Fields whose switches are off will not be updated.""")) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.BULK_EDIT_FORM) - ); + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_EDIT_FORM)); QBackendStepMetaData receiveValuesStep = new QBackendStepMetaData() .withName("receiveValues") @@ -393,7 +395,9 @@ public class QInstanceEnricher .withViewField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, QFieldType.STRING)) .withComponent(new QFrontendComponentMetaData() .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below will be updated if you click Submit.")); + .withValue("text", "The records below will be updated if you click Submit.")) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); QBackendStepMetaData storeStep = new QBackendStepMetaData() .withName("storeRecords") @@ -407,7 +411,9 @@ public class QInstanceEnricher .withViewField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, QFieldType.STRING)) .withComponent(new QFrontendComponentMetaData() .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below have been updated.")); + .withValue("text", "The records below have been updated.")) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); qInstance.addProcess( new QProcessMetaData() @@ -437,7 +443,8 @@ public class QInstanceEnricher .withRecordListFields(new ArrayList<>(table.getFields().values())) .withComponent(new QFrontendComponentMetaData() .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below will be deleted if you click Submit.")); + .withValue("text", "The records below will be deleted if you click Submit.")) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); QBackendStepMetaData storeStep = new QBackendStepMetaData() .withName("delete") @@ -448,7 +455,8 @@ public class QInstanceEnricher .withRecordListFields(new ArrayList<>(table.getFields().values())) .withComponent(new QFrontendComponentMetaData() .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below have been deleted.")); + .withValue("text", "The records below have been deleted.")) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); qInstance.addProcess( new QProcessMetaData() @@ -569,6 +577,7 @@ public class QInstanceEnricher } + /******************************************************************************* ** If a table didn't have any sections, generate "sensible defaults" *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java index a21d7cce..156a984f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java @@ -76,6 +76,17 @@ public class QFieldMetaData + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return ("QFieldMetaData[" + name + "]"); + } + + + /******************************************************************************* ** Initialize a fieldMetaData from a reference to a getter on an entity. ** e.g., new QFieldMetaData(Order::getOrderNo). diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index b2bd3f8c..20b40e02 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -51,6 +51,17 @@ public class QProcessMetaData implements QAppChildMetaData + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return ("QProcessMetaData[" + name + "]"); + } + + + /******************************************************************************* ** Getter for name ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java index 6f70f7e4..08381b0d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java @@ -8,6 +8,7 @@ import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -29,6 +30,32 @@ public abstract class AbstractLoadStep implements BackendStep + /******************************************************************************* + ** Allow subclasses to do an action before the run is complete - before any + ** pages of records are passed in. + *******************************************************************************/ + public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + //////////////////////// + // noop in base class // + //////////////////////// + } + + + + /******************************************************************************* + ** Allow subclasses to do an action after the run is complete - after the last + ** page of records is passed in. + *******************************************************************************/ + public void postRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + //////////////////////// + // noop in base class // + //////////////////////// + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java index 8f97390c..2f9c416e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java @@ -4,6 +4,9 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -16,11 +19,37 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; *******************************************************************************/ public abstract class AbstractTransformStep implements BackendStep { - private List inputRecordPage = new ArrayList<>(); + private List inputRecordPage = new ArrayList<>(); private List outputRecordPage = new ArrayList<>(); + /******************************************************************************* + ** Allow subclasses to do an action before the run is complete - before any + ** pages of records are passed in. + *******************************************************************************/ + public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + //////////////////////// + // noop in base class // + //////////////////////// + } + + + + /******************************************************************************* + ** Allow subclasses to do an action after the run is complete - after the last + ** page of records is passed in. + *******************************************************************************/ + public void postRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + //////////////////////// + // noop in base class // + //////////////////////// + } + + + /******************************************************************************* ** Getter for recordPage ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java index 75c71501..c860176d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java @@ -1,9 +1,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; +import java.util.List; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; /******************************************************************************* @@ -46,4 +51,23 @@ public class BaseStreamedETLStep return (QCodeLoader.getBackendStep(AbstractLoadStep.class, codeReference)); } + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void updateRecordsWithDisplayValuesAndPossibleValues(RunBackendStepInput input, List list) + { + String destinationTable = input.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE); + QTableMetaData table = input.getInstance().getTable(destinationTable); + + if(table != null && list != null) + { + QValueFormatter qValueFormatter = new QValueFormatter(); + qValueFormatter.setDisplayValuesInRecords(table, list); + + QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(input.getInstance(), input.getSession()); + qPossibleValueTranslator.translatePossibleValuesInRecords(input.getTable(), list); + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java index 3a5b1126..aa88dc8d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java @@ -67,6 +67,8 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe AbstractTransformStep transformStep = getTransformStep(runBackendStepInput); AbstractLoadStep loadStep = getLoadStep(runBackendStepInput); + transformStep.preRun(runBackendStepInput, runBackendStepOutput); + transaction = loadStep.openTransaction(runBackendStepInput); loadStep.setTransaction(transaction); @@ -80,15 +82,9 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe ); runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT, recordCount); - runBackendStepOutput.setRecords(loadedRecordList); - ///////////////////// - // commit the work // - ///////////////////// - if(transaction.isPresent()) - { - transaction.get().commit(); - } + updateRecordsWithDisplayValuesAndPossibleValues(runBackendStepInput, loadedRecordList); + runBackendStepOutput.setRecords(loadedRecordList); if(transformStep instanceof ProcessSummaryProviderInterface processSummaryProvider) { @@ -98,6 +94,17 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe ////////////////////////////////////////////////////////////////////////////////////////////// runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummaryProvider.getProcessSummary(true)); } + + transformStep.postRun(runBackendStepInput, runBackendStepOutput); + loadStep.postRun(runBackendStepInput, runBackendStepOutput); + + ///////////////////// + // commit the work // + ///////////////////// + if(transaction.isPresent()) + { + transaction.get().commit(); + } } catch(Exception e) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java index 435480ef..d16c40b5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -90,6 +90,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe extractStep.setRecordPipe(recordPipe); AbstractTransformStep transformStep = getTransformStep(runBackendStepInput); + transformStep.preRun(runBackendStepInput, runBackendStepOutput); List previewRecordList = new ArrayList<>(); new AsyncRecordPipeLoop().run("StreamedETL>Preview>ExtractStep", PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> @@ -100,7 +101,10 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe () -> (consumeRecordsFromPipe(recordPipe, transformStep, runBackendStepInput, runBackendStepOutput, previewRecordList)) ); + updateRecordsWithDisplayValuesAndPossibleValues(runBackendStepInput, previewRecordList); runBackendStepOutput.setRecords(previewRecordList); + + transformStep.postRun(runBackendStepInput, runBackendStepOutput); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java index d5e925d6..58c3780c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java @@ -94,11 +94,13 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back AbstractTransformStep transformStep = getTransformStep(runBackendStepInput); if(!(transformStep instanceof ProcessSummaryProviderInterface processSummaryProvider)) { + // todo - really? if this is required, then put it on the AbstractTransformStep class throw (new QException("Transform Step " + transformStep.getClass().getName() + " does not implement ProcessSummaryProviderInterface.")); } + transformStep.preRun(runBackendStepInput, runBackendStepOutput); List previewRecordList = new ArrayList<>(); - int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Preview>ValidateStep", PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> + int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Preview>ValidateStep", null, recordPipe, (status) -> { extractStep.run(runBackendStepInput, runBackendStepOutput); return (runBackendStepOutput); @@ -106,6 +108,7 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back () -> (consumeRecordsFromPipe(recordPipe, transformStep, runBackendStepInput, runBackendStepOutput, previewRecordList)) ); + updateRecordsWithDisplayValuesAndPossibleValues(runBackendStepInput, previewRecordList); runBackendStepOutput.setRecords(previewRecordList); runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT, recordCount); @@ -113,6 +116,8 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back // get the process summary from the validation step // ////////////////////////////////////////////////////// runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY, processSummaryProvider.getProcessSummary(false)); + + transformStep.postRun(runBackendStepInput, runBackendStepOutput); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 292dee84..49b67354 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -78,6 +78,10 @@ public class StreamedETLWithFrontendProcess public static final String FIELD_VALIDATION_SUMMARY = "validationSummary"; // List public static final String FIELD_PROCESS_SUMMARY = "processResults"; // List + public static final String DEFAULT_PREVIEW_MESSAGE_FOR_INSERT = "This is a preview of the records that will be created."; + public static final String DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE = "This is a preview of the records that will be updated."; + public static final String FIELD_PREVIEW_MESSAGE = "previewMessage"; + /******************************************************************************* @@ -106,6 +110,7 @@ public class StreamedETLWithFrontendProcess ** - FIELD_SUPPORTS_FULL_VALIDATION ** - FIELD_DEFAULT_QUERY_FILTER ** - FIELD_DO_FULL_VALIDATION + ** - FIELD_PREVIEW_MESSAGE *******************************************************************************/ public static QProcessMetaData defineProcessMetaData( Class extractStepClass, @@ -119,10 +124,12 @@ public class StreamedETLWithFrontendProcess .withCode(new QCodeReference(StreamedETLPreviewStep.class)) .withInputData(new QFunctionInputMetaData() .withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE))) + .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE))) .withField(new QFieldMetaData(FIELD_SUPPORTS_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, false))) .withField(new QFieldMetaData(FIELD_DEFAULT_QUERY_FILTER, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DEFAULT_QUERY_FILTER))) .withField(new QFieldMetaData(FIELD_EXTRACT_CODE, QFieldType.STRING).withDefaultValue(new QCodeReference(extractStepClass))) .withField(new QFieldMetaData(FIELD_TRANSFORM_CODE, QFieldType.STRING).withDefaultValue(new QCodeReference(transformStepClass))) + .withField(new QFieldMetaData(FIELD_PREVIEW_MESSAGE, QFieldType.STRING).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_PREVIEW_MESSAGE, DEFAULT_PREVIEW_MESSAGE_FOR_INSERT))) ); QFrontendStepMetaData reviewStep = new QFrontendStepMetaData() @@ -142,7 +149,6 @@ public class StreamedETLWithFrontendProcess .withName(STEP_NAME_EXECUTE) .withCode(new QCodeReference(StreamedETLExecuteStep.class)) .withInputData(new QFunctionInputMetaData() - .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE))) .withField(new QFieldMetaData(FIELD_LOAD_CODE, QFieldType.STRING).withDefaultValue(new QCodeReference(loadStepClass)))) .withOutputMetaData(new QFunctionOutputMetaData() .withField(new QFieldMetaData(FIELD_PROCESS_SUMMARY, QFieldType.STRING)) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java index 08175cdb..d1af5495 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.javalin; import java.io.IOException; import java.io.Serializable; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -443,16 +444,21 @@ public class QJavalinProcessHandler } ProcessState processState = optionalProcessState.get(); - List records = processState.getRecords(); - if(CollectionUtils.nullSafeIsEmpty(records)) + List records = processState.getRecords(); + Map resultForCaller = new HashMap<>(); + + if(records == null) { - throw (new Exception("No records were found for the process.")); + resultForCaller.put("records", new ArrayList<>()); + resultForCaller.put("totalRecords", 0); + } + else + { + List recordPage = CollectionUtils.safelyGetPage(records, skip, limit); + resultForCaller.put("records", recordPage); + resultForCaller.put("totalRecords", records.size()); } - Map resultForCaller = new HashMap<>(); - List recordPage = CollectionUtils.safelyGetPage(records, skip, limit); - resultForCaller.put("records", recordPage); - resultForCaller.put("totalRecords", records.size()); context.result(JsonUtils.toJson(resultForCaller)); } catch(Exception e) diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index ce773b14..63af904e 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -402,6 +402,7 @@ public class SampleMetaDataProvider values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, TABLE_NAME_PERSON); values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, TABLE_NAME_PERSON); values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true); + values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, "This is a preview of what the clones will look like."); QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( ExtractViaQueryStep.class, From f08ffe691fbdcd11d8bf31bbd379b1060e9befb9 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Wed, 31 Aug 2022 12:05:35 -0500 Subject: [PATCH 18/28] PRDONE-94: updated to use interpreter for getting environment credentials, updated interpreter to load Dotenv files as environment overrides --- .../QMetaDataVariableInterpreter.java | 38 ++++++++++++----- .../QMetaDataVariableInterpreterTest.java | 4 +- .../s3/model/metadata/S3BackendMetaData.java | 42 +++++++++++++++++++ .../rdbms/jdbc/ConnectionManagerTest.java | 19 ++++----- .../sampleapp/SampleMetaDataProvider.java | 29 +++++++------ 5 files changed, 95 insertions(+), 37 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java index d3ae8884..7549d047 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java @@ -23,9 +23,12 @@ package com.kingsrook.qqq.backend.core.instances; import java.lang.reflect.Method; +import java.util.HashMap; import java.util.Locale; import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QException; +import io.github.cdimascio.dotenv.Dotenv; +import io.github.cdimascio.dotenv.DotenvEntry; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -45,7 +48,22 @@ public class QMetaDataVariableInterpreter { private static final Logger LOG = LogManager.getLogger(QMetaDataVariableInterpreter.class); - private Map customEnvironment; + private Map environmentOverrides; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QMetaDataVariableInterpreter() + { + environmentOverrides = new HashMap<>(); + Dotenv dotenv = Dotenv.configure().ignoreIfMissing().load(); + for(DotenvEntry e : dotenv.entries()) + { + environmentOverrides.put(e.getKey(), e.getValue()); + } + } @@ -86,7 +104,7 @@ public class QMetaDataVariableInterpreter ////////////////////////////////////////////////////////////////////////////////////////////// // get the value - if it's null, move on, else, interpret it, and put it back in the object // ////////////////////////////////////////////////////////////////////////////////////////////// - Object value = getter.invoke(o); + Object value = getter.invoke(o); if(value == null) { continue; @@ -124,7 +142,7 @@ public class QMetaDataVariableInterpreter if(value.startsWith(envPrefix) && value.endsWith("}")) { String envVarName = value.substring(envPrefix.length()).replaceFirst("}$", ""); - String envValue = getEnvironment().get(envVarName); + String envValue = getEnvironmentVariable(envVarName); return (envValue); } @@ -149,13 +167,13 @@ public class QMetaDataVariableInterpreter /******************************************************************************* - ** Setter for customEnvironment - protected - meant to be called (at least at this + ** Setter for environmentOverrides - protected - meant to be called (at least at this ** time), only in unit test ** *******************************************************************************/ - protected void setCustomEnvironment(Map customEnvironment) + protected void setEnvironmentOverrides(Map environmentOverrides) { - this.customEnvironment = customEnvironment; + this.environmentOverrides = environmentOverrides; } @@ -163,13 +181,13 @@ public class QMetaDataVariableInterpreter /******************************************************************************* ** *******************************************************************************/ - private Map getEnvironment() + private String getEnvironmentVariable(String key) { - if(this.customEnvironment != null) + if(this.environmentOverrides.containsKey(key)) { - return (this.customEnvironment); + return (this.environmentOverrides.get(key)); } - return System.getenv(); + return System.getenv(key); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java index e3881b00..d40a82d9 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java @@ -102,7 +102,7 @@ class QMetaDataVariableInterpreterTest QMetaDataVariableInterpreter secretReader = new QMetaDataVariableInterpreter(); String key = "CUSTOM_PROPERTY"; String value = "ABCD-9876"; - secretReader.setCustomEnvironment(Map.of(key, value)); + secretReader.setEnvironmentOverrides(Map.of(key, value)); assertNull(secretReader.interpret(null)); assertEquals("foo", secretReader.interpret("foo")); @@ -278,4 +278,4 @@ class QMetaDataVariableInterpreterTest } } -} \ No newline at end of file +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java index 92dfccc1..037c9550 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java @@ -50,6 +50,48 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + @Override + public S3BackendMetaData withBasePath(String basePath) + { + setBasePath(basePath); + return this; + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + @Override + public S3BackendMetaData withName(String name) + { + setName(name); + return this; + } + + + + /******************************************************************************* + ** Fluent setter for backendType + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + @Override + public S3BackendMetaData withBackendType(String backendType) + { + setBackendType(backendType); + return this; + } + + + /******************************************************************************* ** Getter for bucketName ** diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java index 0c063dd1..64aa4da9 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java @@ -25,9 +25,8 @@ package com.kingsrook.qqq.backend.module.rdbms.jdbc; import java.sql.Connection; import java.sql.SQLException; import java.util.Collections; -import java.util.Objects; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; -import io.github.cdimascio.dotenv.Dotenv; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -103,13 +102,13 @@ class ConnectionManagerTest private RDBMSBackendMetaData getAuroraBacked() { - Dotenv dotenv = Dotenv.configure().ignoreIfMissing().load(); - String vendor = (dotenv.get("RDBMS_VENDOR") != null) ? dotenv.get("RDBMS_VENDOR") : System.getenv("RDBMS_VENDOR"); - String hostname = (dotenv.get("RDBMS_HOSTNAME") != null) ? dotenv.get("RDBMS_HOSTNAME") : System.getenv("RDBMS_HOSTNAME"); - Integer port = (dotenv.get("RDBMS_PORT") != null) ? Integer.valueOf(Objects.requireNonNull(dotenv.get("RDBMS_PORT"))) : Integer.valueOf(System.getenv("RDBMS_PORT")); - String databaseName = (dotenv.get("RDBMS_DATABASE_NAME") != null) ? dotenv.get("RDBMS_DATABASE_NAME") : System.getenv("RDBMS_DATABASE_NAME"); - String userName = (dotenv.get("RDBMS_USERNAME") != null) ? dotenv.get("RDBMS_USERNAME") : System.getenv("RDBMS_USERNAME"); - String password = (dotenv.get("RDBMS_PASSWORD") != null) ? dotenv.get("RDBMS_PASSWORD") : System.getenv("RDBMS_PASSWORD"); + QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter(); + String vendor = interpreter.interpret("${env.RDBMS_VENDOR}"); + String hostname = interpreter.interpret("${env.RDBMS_HOSTNAME}"); + Integer port = Integer.valueOf(interpreter.interpret("${env.RDBMS_PORT}")); + String databaseName = interpreter.interpret("${env.RDBMS_DATABASE_NAME}"); + String username = interpreter.interpret("${env.RDBMS_USERNAME}"); + String password= interpreter.interpret("${env.RDBMS_PASSWORD}"); return new RDBMSBackendMetaData() .withName("aurora-test") @@ -117,7 +116,7 @@ class ConnectionManagerTest .withHostName(hostname) .withPort(port) .withDatabaseName(databaseName) - .withUsername(userName) + .withUsername(username) .withPassword(password); } } diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index db74876f..1d67b933 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -23,13 +23,13 @@ package com.kingsrook.sampleapp; import java.util.List; -import java.util.Objects; import com.amazonaws.regions.Regions; import com.kingsrook.qqq.backend.core.actions.dashboard.QuickSightChartRenderer; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QValueException; import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; @@ -63,7 +63,6 @@ import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.Filesyst import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.kingsrook.sampleapp.dashboard.widgets.PersonsByCreateDateBarChart; -import io.github.cdimascio.dotenv.Dotenv; /******************************************************************************* @@ -135,11 +134,11 @@ public class SampleMetaDataProvider .withName(PersonsByCreateDateBarChart.class.getSimpleName()) .withCodeReference(new QCodeReference(PersonsByCreateDateBarChart.class, null))); - Dotenv dotenv = Dotenv.configure().ignoreIfMissing().load(); - String accountId = (dotenv.get("QUICKSIGHT_ACCOUNT_ID") != null) ? dotenv.get("QUICKSIGHT_ACCOUNT_ID") : System.getenv("QUICKSIGHT_ACCOUNT_ID"); - String accessKey = (dotenv.get("QUICKSIGHT_ACCESS_KEY") != null) ? dotenv.get("QUICKSIGHT_ACCESS_KEY") : System.getenv("QUICKSIGHT_ACCESS_KEY"); - String secretKey = (dotenv.get("QUICKSIGHT_SECRET_KEY") != null) ? dotenv.get("QUICKSIGHT_SECRET_KEY") : System.getenv("QUICKSIGHT_SECRET_KEY"); - String userArn = (dotenv.get("QUICKSIGHT_USER_ARN") != null) ? dotenv.get("QUICKSIGHT_USER_ARN") : System.getenv("QUICKSIGHT_USER_ARN"); + QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter(); + String accountId = interpreter.interpret("${env.QUICKSIGHT_ACCOUNT_ID}"); + String accessKey = interpreter.interpret("${env.QUICKSIGHT_ACCESS_KEY}"); + String secretKey = interpreter.interpret("${env.QUICKSIGHT_SECRET_KEY}"); + String userArn = interpreter.interpret("${env.QUICKSIGHT_USER_ARN}"); QWidgetMetaDataInterface quickSightChartMetaData = new QuickSightChartMetaData() .withAccountId(accountId) @@ -211,13 +210,13 @@ public class SampleMetaDataProvider { if(USE_MYSQL) { - Dotenv dotenv = Dotenv.configure().ignoreIfMissing().load(); - String vendor = (dotenv.get("RDBMS_VENDOR") != null) ? dotenv.get("RDBMS_VENDOR") : System.getenv("RDBMS_VENDOR"); - String hostname = (dotenv.get("RDBMS_HOSTNAME") != null) ? dotenv.get("RDBMS_HOSTNAME") : System.getenv("RDBMS_HOSTNAME"); - Integer port = (dotenv.get("RDBMS_PORT") != null) ? Integer.valueOf(Objects.requireNonNull(dotenv.get("RDBMS_PORT"))) : Integer.valueOf(System.getenv("RDBMS_PORT")); - String databaseName = (dotenv.get("RDBMS_DATABASE_NAME") != null) ? dotenv.get("RDBMS_DATABASE_NAME") : System.getenv("RDBMS_DATABASE_NAME"); - String userName = (dotenv.get("RDBMS_USERNAME") != null) ? dotenv.get("RDBMS_USERNAME") : System.getenv("RDBMS_USERNAME"); - String password = (dotenv.get("RDBMS_PASSWORD") != null) ? dotenv.get("RDBMS_PASSWORD") : System.getenv("RDBMS_PASSWORD"); + QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter(); + String vendor = interpreter.interpret("${env.RDBMS_VENDOR}"); + String hostname = interpreter.interpret("${env.RDBMS_HOSTNAME}"); + Integer port = Integer.valueOf(interpreter.interpret("${env.RDBMS_PORT}")); + String databaseName = interpreter.interpret("${env.RDBMS_DATABASE_NAME}"); + String username = interpreter.interpret("${env.RDBMS_USERNAME}"); + String password= interpreter.interpret("${env.RDBMS_PASSWORD}"); return new RDBMSBackendMetaData() .withName(RDBMS_BACKEND_NAME) @@ -225,7 +224,7 @@ public class SampleMetaDataProvider .withHostName(hostname) .withPort(port) .withDatabaseName(databaseName) - .withUsername(userName) + .withUsername(username) .withPassword(password); } else From a08ec0ae6fff914d713432d87ae232bdb47de210 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 31 Aug 2022 15:01:09 -0500 Subject: [PATCH 19/28] QQQ-40 Initial working POC --- .../actions/automation/AutomationStatus.java | 78 ++++ .../automation/RecordAutomationHandler.java | 19 + .../RecordAutomationStatusUpdater.java | 96 +++++ .../polling/PollingAutomationExecutor.java | 207 +++++++++++ .../polling/PollingAutomationRunner.java | 240 +++++++++++++ .../core/actions/customizers/QCodeLoader.java | 38 ++ .../core/actions/tables/InsertAction.java | 15 + .../core/actions/tables/UpdateAction.java | 14 + .../core/instances/QInstanceValidator.java | 332 +++++++++++++----- .../automation/RecordAutomationInput.java | 138 ++++++++ .../model/metadata/QAuthenticationType.java | 1 - .../core/model/metadata/QInstance.java | 258 ++++++++------ .../PollingAutomationProviderMetaData.java | 43 +++ .../QAutomationProviderMetaData.java | 77 ++++ .../automation/QAutomationProviderType.java | 60 ++++ .../model/metadata/code/QCodeReference.java | 16 + .../core/model/metadata/code/QCodeUsage.java | 3 +- .../model/metadata/tables/QTableMetaData.java | 38 +- .../automation/AutomationStatusTracking.java | 84 +++++ .../AutomationStatusTrackingType.java | 11 + .../automation/QTableAutomationDetails.java | 133 +++++++ .../automation/TableAutomationAction.java | 232 ++++++++++++ .../tables/automation/TriggerEvent.java | 12 + .../PollingAutomationExecutorTest.java | 233 ++++++++++++ .../instances/QInstanceValidatorTest.java | 237 ++++++++++++- .../qqq/backend/core/utils/TestUtils.java | 182 +++++++++- 26 files changed, 2605 insertions(+), 192 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationHandler.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/automation/RecordAutomationInput.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/PollingAutomationProviderMetaData.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderType.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTracking.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTrackingType.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TableAutomationAction.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TriggerEvent.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java new file mode 100644 index 00000000..ca0b1158 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java @@ -0,0 +1,78 @@ +package com.kingsrook.qqq.backend.core.actions.automation; + + +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; + + +/******************************************************************************* + ** enum of possible values for a record's Automation Status. + *******************************************************************************/ +public enum AutomationStatus implements PossibleValueEnum +{ + PENDING_INSERT_AUTOMATIONS(1, "Pending Insert Automations"), + RUNNING_INSERT_AUTOMATIONS(2, "Running Insert Automations"), + FAILED_INSERT_AUTOMATIONS(3, "Failed Insert Automations"), + PENDING_UPDATE_AUTOMATIONS(4, "Pending Update Automations"), + RUNNING_UPDATE_AUTOMATIONS(5, "Running Update Automations"), + FAILED_UPDATE_AUTOMATIONS(6, "Failed Update Automations"), + OK(7, "OK"); + + + private final Integer id; + private final String label; + + + + /******************************************************************************* + ** + *******************************************************************************/ + AutomationStatus(int id, String label) + { + this.id = id; + this.label = label; + } + + + + /******************************************************************************* + ** Getter for id + ** + *******************************************************************************/ + public Integer getId() + { + return (id); + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return (label); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Integer getPossibleValueId() + { + return (getId()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return (getLabel()); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationHandler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationHandler.java new file mode 100644 index 00000000..74d76fa6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationHandler.java @@ -0,0 +1,19 @@ +package com.kingsrook.qqq.backend.core.actions.automation; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput; + + +/******************************************************************************* + ** Base class for custom-codes to run as an automation action + *******************************************************************************/ +public abstract class RecordAutomationHandler +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public abstract void execute(RecordAutomationInput recordAutomationInput) throws QException; + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java new file mode 100644 index 00000000..8728f4a6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java @@ -0,0 +1,96 @@ +package com.kingsrook.qqq.backend.core.actions.automation; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import org.apache.commons.lang.NotImplementedException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Utility class for updating the automation status data for records + *******************************************************************************/ +public class RecordAutomationStatusUpdater +{ + private static final Logger LOG = LogManager.getLogger(RecordAutomationStatusUpdater.class); + + + + /******************************************************************************* + ** for a list of records from a table, set their automation status - based on + ** how the table is configured. + *******************************************************************************/ + public static boolean setAutomationStatusInRecords(QTableMetaData table, List records, AutomationStatus automationStatus) + { + if(table == null || table.getAutomationDetails() == null || CollectionUtils.nullSafeIsEmpty(records)) + { + return (false); + } + + if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS) || automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS)) + { + Exception e = new Exception(); + for(StackTraceElement stackTraceElement : e.getStackTrace()) + { + String className = stackTraceElement.getClassName(); + if(className.contains("com.kingsrook.qqq.backend.core.actions.automation") && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test")) + { + LOG.info("Avoiding re-setting automation status to PENDING while running an automation"); + return (false); + } + } + } + + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + if(automationDetails.getStatusTracking() != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType())) + { + for(QRecord record : records) + { + record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId()); + // todo - another field - for the automation timestamp?? + } + } + + return (true); + } + + + + /******************************************************************************* + ** for a list of records, update their automation status and actually Update the + ** backend as well. + *******************************************************************************/ + public static void setAutomationStatusInRecordsAndUpdate(QInstance instance, QSession session, QTableMetaData table, List records, AutomationStatus automationStatus) throws QException + { + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + if(automationDetails != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType())) + { + boolean didSetStatusField = setAutomationStatusInRecords(table, records, automationStatus); + if(didSetStatusField) + { + UpdateInput updateInput = new UpdateInput(instance); + updateInput.setSession(session); + updateInput.setTableName(table.getName()); + updateInput.setRecords(records); + new UpdateAction().execute(updateInput); + } + } + else + { + // todo - verify if this is valid as other types are built + throw (new NotImplementedException("Updating record automation status is not implemented for table [" + table + "], tracking type: " + + (automationDetails == null ? "null" : automationDetails.getStatusTracking().getType()))); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java new file mode 100644 index 00000000..205a16e3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java @@ -0,0 +1,207 @@ +package com.kingsrook.qqq.backend.core.actions.automation.polling; + + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Singleton that runs a Polling Automation Provider. Call its 'start' method + ** to make it go. Likely you need to set a sessionSupplier before you start - + ** so that threads that do work will have a valid session. + *******************************************************************************/ +public class PollingAutomationExecutor +{ + private static final Logger LOG = LogManager.getLogger(PollingAutomationExecutor.class); + + private static PollingAutomationExecutor pollingAutomationExecutor = null; + + private Integer initialDelayMillis = 3000; + private Integer delayMillis = 1000; + + private Supplier sessionSupplier; + + private RunningState runningState = RunningState.STOPPED; + private ScheduledExecutorService service; + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private PollingAutomationExecutor() + { + + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static PollingAutomationExecutor getInstance() + { + if(pollingAutomationExecutor == null) + { + pollingAutomationExecutor = new PollingAutomationExecutor(); + } + return (pollingAutomationExecutor); + } + + + + /******************************************************************************* + ** + ** @return true iff the schedule was started + *******************************************************************************/ + public boolean start(QInstance instance, String providerName) + { + if(!runningState.equals(RunningState.STOPPED)) + { + LOG.info("Request to start from an invalid running state [" + runningState + "]. Must be STOPPED."); + return (false); + } + + LOG.info("Starting PollingAutomationExecutor"); + service = Executors.newSingleThreadScheduledExecutor(); + service.scheduleWithFixedDelay(new PollingAutomationRunner(instance, providerName, sessionSupplier), initialDelayMillis, delayMillis, TimeUnit.MILLISECONDS); + runningState = RunningState.RUNNING; + return (true); + } + + + + /******************************************************************************* + ** Stop, and don't wait to check if it worked or anything + *******************************************************************************/ + public void stopAsync() + { + Runnable stopper = this::stop; + stopper.run(); + } + + + + /******************************************************************************* + ** Issue a stop, and wait (a while) for it to succeed. + ** + ** @return true iff we see that the service fully stopped. + *******************************************************************************/ + public boolean stop() + { + if(!runningState.equals(RunningState.RUNNING)) + { + LOG.info("Request to stop from an invalid running state [" + runningState + "]. Must be RUNNING."); + return (false); + } + + LOG.info("Stopping PollingAutomationExecutor"); + runningState = RunningState.STOPPING; + service.shutdown(); + + try + { + if(service.awaitTermination(300, TimeUnit.SECONDS)) + { + LOG.info("Successfully stopped PollingAutomationExecutor"); + runningState = RunningState.STOPPED; + return (true); + } + + LOG.info("Timed out waiting for service to fully terminate. Will be left in STOPPING state."); + } + catch(InterruptedException ie) + { + /////////////////////////////// + // what does this ever mean? // + /////////////////////////////// + } + + return (false); + } + + + + /******************************************************************************* + ** Getter for initialDelayMillis + ** + *******************************************************************************/ + public Integer getInitialDelayMillis() + { + return initialDelayMillis; + } + + + + /******************************************************************************* + ** Setter for initialDelayMillis + ** + *******************************************************************************/ + public void setInitialDelayMillis(Integer initialDelayMillis) + { + this.initialDelayMillis = initialDelayMillis; + } + + + + /******************************************************************************* + ** Getter for delayMillis + ** + *******************************************************************************/ + public Integer getDelayMillis() + { + return delayMillis; + } + + + + /******************************************************************************* + ** Setter for delayMillis + ** + *******************************************************************************/ + public void setDelayMillis(Integer delayMillis) + { + this.delayMillis = delayMillis; + } + + + + /******************************************************************************* + ** Setter for sessionSupplier + ** + *******************************************************************************/ + public void setSessionSupplier(Supplier sessionSupplier) + { + this.sessionSupplier = sessionSupplier; + } + + + + /******************************************************************************* + ** Getter for runningState + ** + *******************************************************************************/ + public RunningState getRunningState() + { + return runningState; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public enum RunningState + { + STOPPED, + RUNNING, + STOPPING, + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java new file mode 100644 index 00000000..46d69208 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java @@ -0,0 +1,240 @@ +package com.kingsrook.qqq.backend.core.actions.automation.polling; + + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; +import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.apache.commons.lang.NotImplementedException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Runnable for the Polling Automation Provider, that looks for records that + ** need automations, and executes them. + *******************************************************************************/ +class PollingAutomationRunner implements Runnable +{ + private static final Logger LOG = LogManager.getLogger(PollingAutomationRunner.class); + + private QInstance instance; + private String providerName; + private Supplier sessionSupplier; + + private List managedTables = new ArrayList<>(); + + private Map> tableInsertActions = new HashMap<>(); + private Map> tableUpdateActions = new HashMap<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public PollingAutomationRunner(QInstance instance, String providerName, Supplier sessionSupplier) + { + this.instance = instance; + this.providerName = providerName; + this.sessionSupplier = sessionSupplier; + + ////////////////////////////////////////////////////////////////////// + // todo - share logic like this among any automation implementation // + ////////////////////////////////////////////////////////////////////// + for(QTableMetaData table : instance.getTables().values()) + { + if(table.getAutomationDetails() != null && this.providerName.equals(table.getAutomationDetails().getProviderName())) + { + managedTables.add(table); + + /////////////////////////////////////////////////////////////////////////// + // organize the table's actions by type // + // todo - in future, need user-defined actions here too (and refreshed!) // + /////////////////////////////////////////////////////////////////////////// + for(TableAutomationAction action : table.getAutomationDetails().getActions()) + { + if(TriggerEvent.POST_INSERT.equals(action.getTriggerEvent())) + { + tableInsertActions.putIfAbsent(table.getName(), new ArrayList<>()); + tableInsertActions.get(table.getName()).add(action); + } + else if(TriggerEvent.POST_UPDATE.equals(action.getTriggerEvent())) + { + tableUpdateActions.putIfAbsent(table.getName(), new ArrayList<>()); + tableUpdateActions.get(table.getName()).add(action); + } + } + + ////////////////////////////// + // sort actions by priority // + ////////////////////////////// + if(tableInsertActions.containsKey(table.getName())) + { + tableInsertActions.get(table.getName()).sort(Comparator.comparing(TableAutomationAction::getPriority)); + } + + if(tableUpdateActions.containsKey(table.getName())) + { + tableUpdateActions.get(table.getName()).sort(Comparator.comparing(TableAutomationAction::getPriority)); + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run() + { + Thread.currentThread().setName(getClass().getSimpleName() + ">" + providerName); + LOG.info("Running " + this.getClass().getSimpleName() + "[providerName=" + providerName + "]"); + + for(QTableMetaData table : managedTables) + { + try + { + processTable(table); + } + catch(Exception e) + { + LOG.error("Error processing automations on table: " + table, e); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void processTable(QTableMetaData table) throws QException + { + QSession session = sessionSupplier != null ? sessionSupplier.get() : new QSession(); + processTableInsertOrUpdate(table, session, true); + processTableInsertOrUpdate(table, session, false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void processTableInsertOrUpdate(QTableMetaData table, QSession session, boolean isInsert) throws QException + { + AutomationStatus automationStatus = isInsert ? AutomationStatus.PENDING_INSERT_AUTOMATIONS : AutomationStatus.PENDING_UPDATE_AUTOMATIONS; + List actions = (isInsert ? tableInsertActions : tableUpdateActions).get(table.getName()); + if(CollectionUtils.nullSafeIsEmpty(actions)) + { + return; + } + + LOG.info(" Query for records " + automationStatus + " in " + table); + + QueryInput queryInput = new QueryInput(instance); + queryInput.setSession(session); // todo - where the heck can we get this from?? + queryInput.setTableName(table.getName()); + + for(TableAutomationAction action : actions) + { + QQueryFilter filter = action.getFilter(); + if(filter == null) + { + filter = new QQueryFilter(); + } + + filter.addCriteria(new QFilterCriteria(table.getAutomationDetails().getStatusTracking().getFieldName(), QCriteriaOperator.IN, List.of(automationStatus.getId()))); + queryInput.setFilter(filter); + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + // todo - pipe this query!! + + if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords())) + { + LOG.info(" Processing " + queryOutput.getRecords().size() + " records in " + table + " for action " + action); + processRecords(table, actions, queryOutput.getRecords(), session, isInsert); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void processRecords(QTableMetaData table, List actions, List records, QSession session, boolean isInsert) throws QException + { + try + { + updateRecordAutomationStatus(table, session, records, isInsert ? AutomationStatus.RUNNING_INSERT_AUTOMATIONS : AutomationStatus.RUNNING_UPDATE_AUTOMATIONS); + + for(TableAutomationAction action : actions) + { + //////////////////////////////////// + // todo - what, re-query them? :( // + //////////////////////////////////// + if(StringUtils.hasContent(action.getProcessName())) + { + ////////////////////////////////////////////////////////////////////////////////////////////// + // todo - uh, how to make these records the input, where an extract step might be involved? // + // should extract step ... see record list and just use it? i think maybe? // + ////////////////////////////////////////////////////////////////////////////////////////////// + throw (new NotImplementedException("processes for automation not yet implemented")); + } + else if(action.getCodeReference() != null) + { + LOG.info(" Executing action: [" + action.getName() + "] as code reference: " + action.getCodeReference()); + RecordAutomationInput input = new RecordAutomationInput(instance); + input.setSession(session); + input.setTableName(table.getName()); + input.setRecordList(records); + + RecordAutomationHandler recordAutomationHandler = QCodeLoader.getRecordAutomationHandler(action); + recordAutomationHandler.execute(input); + } + } + + updateRecordAutomationStatus(table, session, records, AutomationStatus.OK); + } + catch(Exception e) + { + updateRecordAutomationStatus(table, session, records, isInsert ? AutomationStatus.FAILED_INSERT_AUTOMATIONS : AutomationStatus.FAILED_UPDATE_AUTOMATIONS); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void updateRecordAutomationStatus(QTableMetaData table, QSession session, List records, AutomationStatus automationStatus) throws QException + { + RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, automationStatus); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java index f1774928..eec3cab5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java @@ -24,12 +24,14 @@ package com.kingsrook.qqq.backend.core.actions.customizers; import java.util.Optional; import java.util.function.Function; +import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -97,6 +99,42 @@ public class QCodeLoader + /******************************************************************************* + ** + *******************************************************************************/ + public static RecordAutomationHandler getRecordAutomationHandler(TableAutomationAction action) throws QException + { + try + { + QCodeReference codeReference = action.getCodeReference(); + if(!codeReference.getCodeType().equals(QCodeType.JAVA)) + { + /////////////////////////////////////////////////////////////////////////////////////// + // todo - 1) support more languages, 2) wrap them w/ java Functions here, 3) profit! // + /////////////////////////////////////////////////////////////////////////////////////// + throw (new IllegalArgumentException("Only JAVA customizers are supported at this time.")); + } + + Class codeClass = Class.forName(codeReference.getName()); + Object codeObject = codeClass.getConstructor().newInstance(); + if(!(codeObject instanceof RecordAutomationHandler recordAutomationHandler)) + { + throw (new QException("The supplied code [" + codeClass.getName() + "] is not an instance of RecordAutomationHandler")); + } + return (recordAutomationHandler); + } + catch(QException qe) + { + throw (qe); + } + catch(Exception e) + { + throw (new QException("Error getting record automation handler for action [" + action.getName() + "]", e)); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 54ebcfb2..f7dc6869 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.actions.tables; import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; @@ -48,6 +50,9 @@ public class InsertAction *******************************************************************************/ public InsertOutput execute(InsertInput insertInput) throws QException { + ActionHelper.validateSession(insertInput); + setAutomationStatusField(insertInput); + QBackendModuleInterface qModule = getBackendModuleInterface(insertInput); // todo pre-customization - just get to modify the request? InsertOutput insertOutput = qModule.getInsertInterface().execute(insertInput); @@ -57,6 +62,16 @@ public class InsertAction + /******************************************************************************* + ** If the table being inserted into uses an automation-status field, populate it now. + *******************************************************************************/ + private void setAutomationStatusField(InsertInput insertInput) + { + RecordAutomationStatusUpdater.setAutomationStatusInRecords(insertInput.getTable(), insertInput.getRecords(), AutomationStatus.PENDING_INSERT_AUTOMATIONS); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java index 5b8e3e03..4f925411 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java @@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.actions.tables; import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; @@ -42,6 +44,7 @@ public class UpdateAction public UpdateOutput execute(UpdateInput updateInput) throws QException { ActionHelper.validateSession(updateInput); + setAutomationStatusField(updateInput); QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(updateInput.getBackend()); @@ -50,4 +53,15 @@ public class UpdateAction // todo post-customization - can do whatever w/ the result if you want return updateResult; } + + + + /******************************************************************************* + ** If the table being updated uses an automation-status field, populate it now. + *******************************************************************************/ + private void setAutomationStatusField(UpdateInput updateInput) + { + RecordAutomationStatusUpdater.setAutomationStatusInRecords(updateInput.getTable(), updateInput.getRecords(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 4b7c4c2b..3d5a5839 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; @@ -39,10 +40,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import org.apache.logging.log4j.LogManager; @@ -65,6 +70,8 @@ public class QInstanceValidator private boolean printWarnings = false; + private List errors = new ArrayList<>(); + /******************************************************************************* @@ -96,14 +103,14 @@ public class QInstanceValidator ////////////////////////////////////////////////////////////////////////// // do the validation checks - a good qInstance has all conditions TRUE! // ////////////////////////////////////////////////////////////////////////// - List errors = new ArrayList<>(); try { - validateBackends(qInstance, errors); - validateTables(qInstance, errors); - validateProcesses(qInstance, errors); - validateApps(qInstance, errors); - validatePossibleValueSources(qInstance, errors); + validateBackends(qInstance); + validateAutomationProviders(qInstance); + validateTables(qInstance); + validateProcesses(qInstance); + validateApps(qInstance); + validatePossibleValueSources(qInstance); } catch(Exception e) { @@ -123,13 +130,13 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateBackends(QInstance qInstance, List errors) + private void validateBackends(QInstance qInstance) { - if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getBackends()), "At least 1 backend must be defined.")) + if(assertCondition(CollectionUtils.nullSafeHasContents(qInstance.getBackends()), "At least 1 backend must be defined.")) { qInstance.getBackends().forEach((backendName, backend) -> { - assertCondition(errors, Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + "."); + assertCondition(Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + "."); }); } } @@ -139,42 +146,59 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateTables(QInstance qInstance, List errors) + private void validateAutomationProviders(QInstance qInstance) { - if(assertCondition(errors, CollectionUtils.nullSafeHasContents(qInstance.getTables()), + if(qInstance.getAutomationProviders() != null) + { + qInstance.getAutomationProviders().forEach((name, automationProvider) -> + { + assertCondition(Objects.equals(name, automationProvider.getName()), "Inconsistent naming for automationProvider: " + name + "/" + automationProvider.getName() + "."); + assertCondition(automationProvider.getType() != null, "Missing type for automationProvider: " + name); + }); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void validateTables(QInstance qInstance) + { + if(assertCondition(CollectionUtils.nullSafeHasContents(qInstance.getTables()), "At least 1 table must be defined.")) { qInstance.getTables().forEach((tableName, table) -> { - assertCondition(errors, Objects.equals(tableName, table.getName()), "Inconsistent naming for table: " + tableName + "/" + table.getName() + "."); + assertCondition(Objects.equals(tableName, table.getName()), "Inconsistent naming for table: " + tableName + "/" + table.getName() + "."); - validateAppChildHasValidParentAppName(qInstance, errors, table); + validateAppChildHasValidParentAppName(qInstance, table); //////////////////////////////////////// // validate the backend for the table // //////////////////////////////////////// - if(assertCondition(errors, StringUtils.hasContent(table.getBackendName()), + if(assertCondition(StringUtils.hasContent(table.getBackendName()), "Missing backend name for table " + tableName + ".")) { if(CollectionUtils.nullSafeHasContents(qInstance.getBackends())) { - assertCondition(errors, qInstance.getBackendForTable(tableName) != null, "Unrecognized backend " + table.getBackendName() + " for table " + tableName + "."); + assertCondition(qInstance.getBackendForTable(tableName) != null, "Unrecognized backend " + table.getBackendName() + " for table " + tableName + "."); } } ////////////////////////////////// // validate fields in the table // ////////////////////////////////// - if(assertCondition(errors, CollectionUtils.nullSafeHasContents(table.getFields()), "At least 1 field must be defined in table " + tableName + ".")) + if(assertCondition(CollectionUtils.nullSafeHasContents(table.getFields()), "At least 1 field must be defined in table " + tableName + ".")) { table.getFields().forEach((fieldName, field) -> { - assertCondition(errors, Objects.equals(fieldName, field.getName()), + assertCondition(Objects.equals(fieldName, field.getName()), "Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + "."); if(field.getPossibleValueSourceName() != null) { - assertCondition(errors, qInstance.getPossibleValueSource(field.getPossibleValueSourceName()) != null, + assertCondition(qInstance.getPossibleValueSource(field.getPossibleValueSourceName()) != null, "Unrecognized possibleValueSourceName " + field.getPossibleValueSourceName() + " in table " + tableName + " for field " + fieldName + "."); } }); @@ -189,10 +213,10 @@ public class QInstanceValidator { for(QFieldSection section : table.getSections()) { - validateSection(errors, table, section, fieldNamesInSections); + validateSection(table, section, fieldNamesInSections); if(section.getTier().equals(Tier.T1)) { - assertCondition(errors, tier1Section == null, "Table " + tableName + " has more than 1 section listed as Tier 1"); + assertCondition(tier1Section == null, "Table " + tableName + " has more than 1 section listed as Tier 1"); tier1Section = section; } } @@ -202,7 +226,7 @@ public class QInstanceValidator { for(String fieldName : table.getFields().keySet()) { - assertCondition(errors, fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is not listed in any field sections."); + assertCondition(fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is not listed in any field sections."); } } @@ -213,7 +237,7 @@ public class QInstanceValidator { for(String recordLabelField : table.getRecordLabelFields()) { - assertCondition(errors, table.getFields().containsKey(recordLabelField), "Table " + tableName + " record label field " + recordLabelField + " is not a field on this table."); + assertCondition(table.getFields().containsKey(recordLabelField), "Table " + tableName + " record label field " + recordLabelField + " is not a field on this table."); } } @@ -221,9 +245,17 @@ public class QInstanceValidator { for(Map.Entry entry : table.getCustomizers().entrySet()) { - validateTableCustomizer(errors, tableName, entry.getKey(), entry.getValue()); + validateTableCustomizer(tableName, entry.getKey(), entry.getValue()); } } + + ////////////////////////////////////// + // validate the table's automations // + ////////////////////////////////////// + if(table.getAutomationDetails() != null) + { + validateTableAutomationDetails(qInstance, table); + } }); } } @@ -233,11 +265,112 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateTableCustomizer(List errors, String tableName, String customizerName, QCodeReference codeReference) + private void validateTableAutomationDetails(QInstance qInstance, QTableMetaData table) + { + String tableName = table.getName(); + String prefix = "Table " + tableName + " automationDetails "; + + QTableAutomationDetails automationDetails = table.getAutomationDetails(); + + ////////////////////////////////////// + // validate the automation provider // + ////////////////////////////////////// + String providerName = automationDetails.getProviderName(); + if(assertCondition(StringUtils.hasContent(providerName), prefix + " is missing a providerName")) + { + assertCondition(qInstance.getAutomationProvider(providerName) != null, " has an unrecognized providerName: " + providerName); + } + + ////////////////////////////////// + // validate the status tracking // + ////////////////////////////////// + AutomationStatusTracking statusTracking = automationDetails.getStatusTracking(); + if(assertCondition(statusTracking != null, prefix + "do not have statusTracking defined.")) + { + if(assertCondition(statusTracking.getType() != null, prefix + "statusTracking is missing a type")) + { + if(statusTracking.getType().equals(AutomationStatusTrackingType.FIELD_IN_TABLE)) + { + assertCondition(StringUtils.hasContent(statusTracking.getFieldName()), prefix + "statusTracking of type fieldInTable is missing its fieldName"); + } + } + } + + ////////////////////////// + // validate the actions // + ////////////////////////// + Set usedNames = new HashSet<>(); + if(automationDetails.getActions() != null) + { + automationDetails.getActions().forEach(action -> + { + assertCondition(StringUtils.hasContent(action.getName()), prefix + "has an action missing a name"); + assertCondition(!usedNames.contains(action.getName()), prefix + "has more than one action named " + action.getName()); + usedNames.add(action.getName()); + + String actionPrefix = prefix + "action " + action.getName() + " "; + assertCondition(action.getTriggerEvent() != null, actionPrefix + "is missing a triggerEvent"); + + ///////////////////////////////////////////////////// + // validate the code or process used by the action // + ///////////////////////////////////////////////////// + int numberSet = 0; + if(action.getCodeReference() != null) + { + numberSet++; + if(preAssertionsForCodeReference(action.getCodeReference(), actionPrefix)) + { + validateSimpleCodeReference(actionPrefix + "code reference: ", action.getCodeReference(), RecordAutomationHandler.class); + } + } + + if(action.getProcessName() != null) + { + numberSet++; + QProcessMetaData process = qInstance.getProcess(action.getProcessName()); + if(assertCondition(process != null, actionPrefix + "has an unrecognized processName: " + action.getProcessName())) + { + if(process.getTableName() != null) + { + assertCondition(tableName.equals(process.getTableName()), actionPrefix + " references a process from a different table"); + } + } + } + + assertCondition(numberSet != 0, actionPrefix + "is missing both a codeReference and a processName"); + assertCondition(!(numberSet > 1), actionPrefix + "has both a codeReference and a processName (which is not allowed)"); + + /////////////////////////////////////////// + // validate the filter (if there is one) // + /////////////////////////////////////////// + if(action.getFilter() != null && action.getFilter().getCriteria() != null) + { + action.getFilter().getCriteria().forEach((criteria) -> + { + if(assertCondition(StringUtils.hasContent(criteria.getFieldName()), actionPrefix + "has a filter criteria without a field name")) + { + assertNoException(() -> table.getField(criteria.getFieldName()), actionPrefix + "has a filter criteria referencing an unrecognized field: " + criteria.getFieldName()); + } + + assertCondition(criteria.getOperator() != null, actionPrefix + "has a filter criteria without an operator"); + + // todo - validate cardinality of values... + }); + } + }); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void validateTableCustomizer(String tableName, String customizerName, QCodeReference codeReference) { String prefix = "Table " + tableName + ", customizer " + customizerName + ": "; - if(!preAssertionsForCodeReference(errors, codeReference, prefix)) + if(!preAssertionsForCodeReference(codeReference, prefix)) { return; } @@ -245,18 +378,18 @@ public class QInstanceValidator ////////////////////////////////////////////////////////////////////////////// // make sure (at this time) that it's a java type, then do some java checks // ////////////////////////////////////////////////////////////////////////////// - if(assertCondition(errors, codeReference.getCodeType().equals(QCodeType.JAVA), prefix + "Only JAVA customizers are supported at this time.")) + if(assertCondition(codeReference.getCodeType().equals(QCodeType.JAVA), prefix + "Only JAVA customizers are supported at this time.")) { /////////////////////////////////////// // make sure the class can be loaded // /////////////////////////////////////// - Class customizerClass = getClassForCodeReference(errors, codeReference, prefix); + Class customizerClass = getClassForCodeReference(codeReference, prefix); if(customizerClass != null) { ////////////////////////////////////////////////// // make sure the customizer can be instantiated // ////////////////////////////////////////////////// - Object customizerInstance = getInstanceOfCodeReference(errors, prefix, customizerClass); + Object customizerInstance = getInstanceOfCodeReference(prefix, customizerClass); TableCustomizers tableCustomizer = TableCustomizers.forRole(customizerName); if(tableCustomizer == null) @@ -273,7 +406,7 @@ public class QInstanceValidator //////////////////////////////////////////////////////////////////////// if(customizerInstance != null && tableCustomizer.getTableCustomizer().getExpectedType() != null) { - Object castedObject = getCastedObject(errors, prefix, tableCustomizer.getTableCustomizer().getExpectedType(), customizerInstance); + Object castedObject = getCastedObject(prefix, tableCustomizer.getTableCustomizer().getExpectedType(), customizerInstance); Consumer validationFunction = tableCustomizer.getTableCustomizer().getValidationFunction(); if(castedObject != null && validationFunction != null) @@ -305,7 +438,7 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private T getCastedObject(List errors, String prefix, Class expectedType, Object customizerInstance) + private T getCastedObject(String prefix, Class expectedType, Object customizerInstance) { T castedObject = null; try @@ -314,7 +447,7 @@ public class QInstanceValidator } catch(ClassCastException e) { - errors.add(prefix + "CodeReference could not be casted to the expected type: " + expectedType); + errors.add(prefix + "CodeReference is not of the expected type: " + expectedType); } return castedObject; } @@ -324,7 +457,7 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private Object getInstanceOfCodeReference(List errors, String prefix, Class customizerClass) + private Object getInstanceOfCodeReference(String prefix, Class customizerClass) { Object customizerInstance = null; try @@ -343,18 +476,18 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateSection(List errors, QTableMetaData table, QFieldSection section, Set fieldNamesInSections) + private void validateSection(QTableMetaData table, QFieldSection section, Set fieldNamesInSections) { - assertCondition(errors, StringUtils.hasContent(section.getName()), "Missing a name for field section in table " + table.getName() + "."); - assertCondition(errors, StringUtils.hasContent(section.getLabel()), "Missing a label for field section in table " + table.getLabel() + "."); - if(assertCondition(errors, CollectionUtils.nullSafeHasContents(section.getFieldNames()), "Table " + table.getName() + " section " + section.getName() + " does not have any fields.")) + assertCondition(StringUtils.hasContent(section.getName()), "Missing a name for field section in table " + table.getName() + "."); + assertCondition(StringUtils.hasContent(section.getLabel()), "Missing a label for field section in table " + table.getLabel() + "."); + if(assertCondition(CollectionUtils.nullSafeHasContents(section.getFieldNames()), "Table " + table.getName() + " section " + section.getName() + " does not have any fields.")) { if(table.getFields() != null) { for(String fieldName : section.getFieldNames()) { - assertCondition(errors, table.getFields().containsKey(fieldName), "Table " + table.getName() + " section " + section.getName() + " specifies fieldName " + fieldName + ", which is not a field on this table."); - assertCondition(errors, !fieldNamesInSections.contains(fieldName), "Table " + table.getName() + " has field " + fieldName + " listed more than once in its field sections."); + assertCondition(table.getFields().containsKey(fieldName), "Table " + table.getName() + " section " + section.getName() + " specifies fieldName " + fieldName + ", which is not a field on this table."); + assertCondition(!fieldNamesInSections.contains(fieldName), "Table " + table.getName() + " has field " + fieldName + " listed more than once in its field sections."); fieldNamesInSections.add(fieldName); } @@ -367,33 +500,33 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateProcesses(QInstance qInstance, List errors) + private void validateProcesses(QInstance qInstance) { if(CollectionUtils.nullSafeHasContents(qInstance.getProcesses())) { qInstance.getProcesses().forEach((processName, process) -> { - assertCondition(errors, Objects.equals(processName, process.getName()), "Inconsistent naming for process: " + processName + "/" + process.getName() + "."); + assertCondition(Objects.equals(processName, process.getName()), "Inconsistent naming for process: " + processName + "/" + process.getName() + "."); - validateAppChildHasValidParentAppName(qInstance, errors, process); + validateAppChildHasValidParentAppName(qInstance, process); ///////////////////////////////////////////// // validate the table name for the process // ///////////////////////////////////////////// if(process.getTableName() != null) { - assertCondition(errors, qInstance.getTable(process.getTableName()) != null, "Unrecognized table " + process.getTableName() + " for process " + processName + "."); + assertCondition(qInstance.getTable(process.getTableName()) != null, "Unrecognized table " + process.getTableName() + " for process " + processName + "."); } /////////////////////////////////// // validate steps in the process // /////////////////////////////////// - if(assertCondition(errors, CollectionUtils.nullSafeHasContents(process.getStepList()), "At least 1 step must be defined in process " + processName + ".")) + if(assertCondition(CollectionUtils.nullSafeHasContents(process.getStepList()), "At least 1 step must be defined in process " + processName + ".")) { int index = 0; for(QStepMetaData step : process.getStepList()) { - assertCondition(errors, StringUtils.hasContent(step.getName()), "Missing name for a step at index " + index + " in process " + processName); + assertCondition(StringUtils.hasContent(step.getName()), "Missing name for a step at index " + index + " in process " + processName); index++; } } @@ -406,26 +539,26 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateApps(QInstance qInstance, List errors) + private void validateApps(QInstance qInstance) { if(CollectionUtils.nullSafeHasContents(qInstance.getApps())) { qInstance.getApps().forEach((appName, app) -> { - assertCondition(errors, Objects.equals(appName, app.getName()), "Inconsistent naming for app: " + appName + "/" + app.getName() + "."); + assertCondition(Objects.equals(appName, app.getName()), "Inconsistent naming for app: " + appName + "/" + app.getName() + "."); - validateAppChildHasValidParentAppName(qInstance, errors, app); + validateAppChildHasValidParentAppName(qInstance, app); Set appsVisited = new HashSet<>(); - visitAppCheckingForCycles(app, appsVisited, errors); + visitAppCheckingForCycles(app, appsVisited); if(app.getChildren() != null) { Set childNames = new HashSet<>(); for(QAppChildMetaData child : app.getChildren()) { - assertCondition(errors, Objects.equals(appName, child.getParentAppName()), "Child " + child.getName() + " of app " + appName + " does not have its parent app properly set."); - assertCondition(errors, !childNames.contains(child.getName()), "App " + appName + " contains more than one child named " + child.getName()); + assertCondition(Objects.equals(appName, child.getParentAppName()), "Child " + child.getName() + " of app " + appName + " does not have its parent app properly set."); + assertCondition(!childNames.contains(child.getName()), "App " + appName + " contains more than one child named " + child.getName()); childNames.add(child.getName()); } } @@ -438,14 +571,14 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validatePossibleValueSources(QInstance qInstance, List errors) + private void validatePossibleValueSources(QInstance qInstance) { if(CollectionUtils.nullSafeHasContents(qInstance.getPossibleValueSources())) { qInstance.getPossibleValueSources().forEach((pvsName, possibleValueSource) -> { - assertCondition(errors, Objects.equals(pvsName, possibleValueSource.getName()), "Inconsistent naming for possibleValueSource: " + pvsName + "/" + possibleValueSource.getName() + "."); - if(assertCondition(errors, possibleValueSource.getType() != null, "Missing type for possibleValueSource: " + pvsName)) + assertCondition(Objects.equals(pvsName, possibleValueSource.getName()), "Inconsistent naming for possibleValueSource: " + pvsName + "/" + possibleValueSource.getName() + "."); + if(assertCondition(possibleValueSource.getType() != null, "Missing type for possibleValueSource: " + pvsName)) { //////////////////////////////////////////////////////////////////////////////////////////////// // assert about fields that should and should not be set, based on possible value source type // @@ -455,30 +588,30 @@ public class QInstanceValidator { case ENUM -> { - assertCondition(errors, !StringUtils.hasContent(possibleValueSource.getTableName()), "enum-type possibleValueSource " + pvsName + " should not have a tableName."); - assertCondition(errors, possibleValueSource.getCustomCodeReference() == null, "enum-type possibleValueSource " + pvsName + " should not have a customCodeReference."); + assertCondition(!StringUtils.hasContent(possibleValueSource.getTableName()), "enum-type possibleValueSource " + pvsName + " should not have a tableName."); + assertCondition(possibleValueSource.getCustomCodeReference() == null, "enum-type possibleValueSource " + pvsName + " should not have a customCodeReference."); - assertCondition(errors, CollectionUtils.nullSafeHasContents(possibleValueSource.getEnumValues()), "enum-type possibleValueSource " + pvsName + " is missing enum values"); + assertCondition(CollectionUtils.nullSafeHasContents(possibleValueSource.getEnumValues()), "enum-type possibleValueSource " + pvsName + " is missing enum values"); } case TABLE -> { - assertCondition(errors, CollectionUtils.nullSafeIsEmpty(possibleValueSource.getEnumValues()), "table-type possibleValueSource " + pvsName + " should not have enum values."); - assertCondition(errors, possibleValueSource.getCustomCodeReference() == null, "table-type possibleValueSource " + pvsName + " should not have a customCodeReference."); + assertCondition(CollectionUtils.nullSafeIsEmpty(possibleValueSource.getEnumValues()), "table-type possibleValueSource " + pvsName + " should not have enum values."); + assertCondition(possibleValueSource.getCustomCodeReference() == null, "table-type possibleValueSource " + pvsName + " should not have a customCodeReference."); - if(assertCondition(errors, StringUtils.hasContent(possibleValueSource.getTableName()), "table-type possibleValueSource " + pvsName + " is missing a tableName.")) + if(assertCondition(StringUtils.hasContent(possibleValueSource.getTableName()), "table-type possibleValueSource " + pvsName + " is missing a tableName.")) { - assertCondition(errors, qInstance.getTable(possibleValueSource.getTableName()) != null, "Unrecognized table " + possibleValueSource.getTableName() + " for possibleValueSource " + pvsName + "."); + assertCondition(qInstance.getTable(possibleValueSource.getTableName()) != null, "Unrecognized table " + possibleValueSource.getTableName() + " for possibleValueSource " + pvsName + "."); } } case CUSTOM -> { - assertCondition(errors, CollectionUtils.nullSafeIsEmpty(possibleValueSource.getEnumValues()), "custom-type possibleValueSource " + pvsName + " should not have enum values."); - assertCondition(errors, !StringUtils.hasContent(possibleValueSource.getTableName()), "custom-type possibleValueSource " + pvsName + " should not have a tableName."); + assertCondition(CollectionUtils.nullSafeIsEmpty(possibleValueSource.getEnumValues()), "custom-type possibleValueSource " + pvsName + " should not have enum values."); + assertCondition(!StringUtils.hasContent(possibleValueSource.getTableName()), "custom-type possibleValueSource " + pvsName + " should not have a tableName."); - if(assertCondition(errors, possibleValueSource.getCustomCodeReference() != null, "custom-type possibleValueSource " + pvsName + " is missing a customCodeReference.")) + if(assertCondition(possibleValueSource.getCustomCodeReference() != null, "custom-type possibleValueSource " + pvsName + " is missing a customCodeReference.")) { - assertCondition(errors, QCodeUsage.POSSIBLE_VALUE_PROVIDER.equals(possibleValueSource.getCustomCodeReference().getCodeUsage()), "customCodeReference for possibleValueSource " + pvsName + " is not a possibleValueProvider."); - validateCustomPossibleValueSourceCode(errors, pvsName, possibleValueSource.getCustomCodeReference()); + assertCondition(QCodeUsage.POSSIBLE_VALUE_PROVIDER.equals(possibleValueSource.getCustomCodeReference().getCodeUsage()), "customCodeReference for possibleValueSource " + pvsName + " is not a possibleValueProvider."); + validateSimpleCodeReference("PossibleValueSource " + pvsName + " custom code reference: ", possibleValueSource.getCustomCodeReference(), QCustomPossibleValueProvider.class); } } default -> errors.add("Unexpected possibleValueSource type: " + possibleValueSource.getType()); @@ -493,11 +626,9 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateCustomPossibleValueSourceCode(List errors, String pvsName, QCodeReference codeReference) + private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class expectedClass) { - String prefix = "PossibleValueSource " + pvsName + " custom code reference: "; - - if(!preAssertionsForCodeReference(errors, codeReference, prefix)) + if(!preAssertionsForCodeReference(codeReference, prefix)) { return; } @@ -505,25 +636,25 @@ public class QInstanceValidator ////////////////////////////////////////////////////////////////////////////// // make sure (at this time) that it's a java type, then do some java checks // ////////////////////////////////////////////////////////////////////////////// - if(assertCondition(errors, codeReference.getCodeType().equals(QCodeType.JAVA), prefix + "Only JAVA customizers are supported at this time.")) + if(assertCondition(codeReference.getCodeType().equals(QCodeType.JAVA), prefix + "Only JAVA code references are supported at this time.")) { /////////////////////////////////////// // make sure the class can be loaded // /////////////////////////////////////// - Class customizerClass = getClassForCodeReference(errors, codeReference, prefix); + Class customizerClass = getClassForCodeReference(codeReference, prefix); if(customizerClass != null) { ////////////////////////////////////////////////// // make sure the customizer can be instantiated // ////////////////////////////////////////////////// - Object customizerInstance = getInstanceOfCodeReference(errors, prefix, customizerClass); + Object customizerInstance = getInstanceOfCodeReference(prefix, customizerClass); //////////////////////////////////////////////////////////////////////// // make sure the customizer instance can be cast to the expected type // //////////////////////////////////////////////////////////////////////// if(customizerInstance != null) { - getCastedObject(errors, prefix, QCustomPossibleValueProvider.class, customizerInstance); + getCastedObject(prefix, expectedClass, customizerInstance); } } } @@ -534,7 +665,7 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private Class getClassForCodeReference(List errors, QCodeReference codeReference, String prefix) + private Class getClassForCodeReference(QCodeReference codeReference, String prefix) { Class customizerClass = null; try @@ -553,15 +684,15 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private boolean preAssertionsForCodeReference(List errors, QCodeReference codeReference, String prefix) + private boolean preAssertionsForCodeReference(QCodeReference codeReference, String prefix) { boolean okay = true; - if(!assertCondition(errors, StringUtils.hasContent(codeReference.getName()), prefix + " is missing a code reference name")) + if(!assertCondition(StringUtils.hasContent(codeReference.getName()), prefix + " is missing a code reference name")) { okay = false; } - if(!assertCondition(errors, codeReference.getCodeType() != null, prefix + " is missing a code type")) + if(!assertCondition(codeReference.getCodeType() != null, prefix + " is missing a code type")) { okay = false; } @@ -575,9 +706,9 @@ public class QInstanceValidator ** Check if an app's child list can recursively be traversed without finding a ** duplicate, which would indicate a cycle (e.g., an error) *******************************************************************************/ - private void visitAppCheckingForCycles(QAppMetaData app, Set appsVisited, List errors) + private void visitAppCheckingForCycles(QAppMetaData app, Set appsVisited) { - if(assertCondition(errors, !appsVisited.contains(app.getName()), "Circular app reference detected, involving " + app.getName())) + if(assertCondition(!appsVisited.contains(app.getName()), "Circular app reference detected, involving " + app.getName())) { appsVisited.add(app.getName()); if(app.getChildren() != null) @@ -586,7 +717,7 @@ public class QInstanceValidator { if(child instanceof QAppMetaData childApp) { - visitAppCheckingForCycles(childApp, appsVisited, errors); + visitAppCheckingForCycles(childApp, appsVisited); } } } @@ -598,11 +729,11 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateAppChildHasValidParentAppName(QInstance qInstance, List errors, QAppChildMetaData appChild) + private void validateAppChildHasValidParentAppName(QInstance qInstance, QAppChildMetaData appChild) { if(appChild.getParentAppName() != null) { - assertCondition(errors, qInstance.getApp(appChild.getParentAppName()) != null, "Unrecognized parent app " + appChild.getParentAppName() + " for " + appChild.getName() + "."); + assertCondition(qInstance.getApp(appChild.getParentAppName()) != null, "Unrecognized parent app " + appChild.getParentAppName() + " for " + appChild.getName() + "."); } } @@ -613,7 +744,7 @@ public class QInstanceValidator ** But if it's false, add the provided message to the list of errors (and return false, ** e.g., in case you need to stop evaluating rules to avoid exceptions). *******************************************************************************/ - private boolean assertCondition(List errors, boolean condition, String message) + private boolean assertCondition(boolean condition, String message) { if(!condition) { @@ -625,6 +756,41 @@ public class QInstanceValidator + /******************************************************************************* + ** For the given lambda, if it doesn't throw an exception, then we're all good (and return true). + ** But if it throws, add the provided message to the list of errors (and return false, + ** e.g., in case you need to stop evaluating rules to avoid exceptions). + *******************************************************************************/ + private boolean assertNoException(UnsafeLambda unsafeLambda, String message) + { + try + { + unsafeLambda.run(); + return (true); + } + catch(Exception e) + { + errors.add(message); + return (false); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @FunctionalInterface + interface UnsafeLambda + { + /******************************************************************************* + ** + *******************************************************************************/ + void run() throws Exception; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/automation/RecordAutomationInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/automation/RecordAutomationInput.java new file mode 100644 index 00000000..d3cd1c4d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/automation/RecordAutomationInput.java @@ -0,0 +1,138 @@ +package com.kingsrook.qqq.backend.core.model.automation; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; + + +/******************************************************************************* + ** Input data for the RecordAutomationHandler interface. + *******************************************************************************/ +public class RecordAutomationInput extends AbstractTableActionInput +{ + private TableAutomationAction action; + + //////////////////////////////////////////// + // todo - why both? pick one? or don't? // + // maybe - if recordList is null and primaryKeyList isn't, then do the record query in here? + //////////////////////////////////////////// + private List recordList; + private List primaryKeyList; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public RecordAutomationInput(QInstance instance) + { + super(instance); + } + + + + /******************************************************************************* + ** Getter for action + ** + *******************************************************************************/ + public TableAutomationAction getAction() + { + return action; + } + + + + /******************************************************************************* + ** Setter for action + ** + *******************************************************************************/ + public void setAction(TableAutomationAction action) + { + this.action = action; + } + + + + /******************************************************************************* + ** Fluent setter for action + ** + *******************************************************************************/ + public RecordAutomationInput withAction(TableAutomationAction action) + { + this.action = action; + return (this); + } + + + + /******************************************************************************* + ** Getter for recordList + ** + *******************************************************************************/ + public List getRecordList() + { + return recordList; + } + + + + /******************************************************************************* + ** Setter for recordList + ** + *******************************************************************************/ + public void setRecordList(List recordList) + { + this.recordList = recordList; + } + + + + /******************************************************************************* + ** Fluent setter for recordList + ** + *******************************************************************************/ + public RecordAutomationInput withRecordList(List recordList) + { + this.recordList = recordList; + return (this); + } + + + + /******************************************************************************* + ** Getter for primaryKeyList + ** + *******************************************************************************/ + public List getPrimaryKeyList() + { + return primaryKeyList; + } + + + + /******************************************************************************* + ** Setter for primaryKeyList + ** + *******************************************************************************/ + public void setPrimaryKeyList(List primaryKeyList) + { + this.primaryKeyList = primaryKeyList; + } + + + + /******************************************************************************* + ** Fluent setter for primaryKeyList + ** + *******************************************************************************/ + public RecordAutomationInput withPrimaryKeyList(List primaryKeyList) + { + this.primaryKeyList = primaryKeyList; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QAuthenticationType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QAuthenticationType.java index afc24aaa..0550ee66 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QAuthenticationType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QAuthenticationType.java @@ -26,7 +26,6 @@ package com.kingsrook.qqq.backend.core.model.metadata; ** Enum to define the possible authentication types ** *******************************************************************************/ -@SuppressWarnings("rawtypes") public enum QAuthenticationType { AUTH_0("auth0"), diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 3712d077..95faf73c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.instances.QInstanceValidationKey; +import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; @@ -49,7 +50,8 @@ public class QInstance @JsonIgnore private Map backends = new HashMap<>(); - private QAuthenticationMetaData authentication = null; + private QAuthenticationMetaData authentication = null; + private Map automationProviders = new HashMap<>(); //////////////////////////////////////////////////////////////////////////////////////////// // Important to use LinkedHashmap here, to preserve the order in which entries are added. // @@ -103,17 +105,6 @@ public class QInstance - /******************************************************************************* - ** Setter for hasBeenValidated - ** - *******************************************************************************/ - public void setHasBeenValidated(QInstanceValidationKey key) - { - this.hasBeenValidated = true; - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -148,6 +139,28 @@ public class QInstance + /******************************************************************************* + ** Getter for backends + ** + *******************************************************************************/ + public Map getBackends() + { + return backends; + } + + + + /******************************************************************************* + ** Setter for backends + ** + *******************************************************************************/ + public void setBackends(Map backends) + { + this.backends = backends; + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -187,6 +200,28 @@ public class QInstance + /******************************************************************************* + ** Getter for tables + ** + *******************************************************************************/ + public Map getTables() + { + return tables; + } + + + + /******************************************************************************* + ** Setter for tables + ** + *******************************************************************************/ + public void setTables(Map tables) + { + this.tables = tables; + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -221,6 +256,28 @@ public class QInstance + /******************************************************************************* + ** Getter for possibleValueSources + ** + *******************************************************************************/ + public Map getPossibleValueSources() + { + return possibleValueSources; + } + + + + /******************************************************************************* + ** Setter for possibleValueSources + ** + *******************************************************************************/ + public void setPossibleValueSources(Map possibleValueSources) + { + this.possibleValueSources = possibleValueSources; + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -271,6 +328,28 @@ public class QInstance + /******************************************************************************* + ** Getter for processes + ** + *******************************************************************************/ + public Map getProcesses() + { + return processes; + } + + + + /******************************************************************************* + ** Setter for processes + ** + *******************************************************************************/ + public void setProcesses(Map processes) + { + this.processes = processes; + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -305,94 +384,6 @@ public class QInstance - /******************************************************************************* - ** Getter for backends - ** - *******************************************************************************/ - public Map getBackends() - { - return backends; - } - - - - /******************************************************************************* - ** Setter for backends - ** - *******************************************************************************/ - public void setBackends(Map backends) - { - this.backends = backends; - } - - - - /******************************************************************************* - ** Getter for tables - ** - *******************************************************************************/ - public Map getTables() - { - return tables; - } - - - - /******************************************************************************* - ** Setter for tables - ** - *******************************************************************************/ - public void setTables(Map tables) - { - this.tables = tables; - } - - - - /******************************************************************************* - ** Getter for possibleValueSources - ** - *******************************************************************************/ - public Map getPossibleValueSources() - { - return possibleValueSources; - } - - - - /******************************************************************************* - ** Setter for possibleValueSources - ** - *******************************************************************************/ - public void setPossibleValueSources(Map possibleValueSources) - { - this.possibleValueSources = possibleValueSources; - } - - - - /******************************************************************************* - ** Getter for processes - ** - *******************************************************************************/ - public Map getProcesses() - { - return processes; - } - - - - /******************************************************************************* - ** Setter for processes - ** - *******************************************************************************/ - public void setProcesses(Map processes) - { - this.processes = processes; - } - - - /******************************************************************************* ** Getter for apps ** @@ -415,6 +406,62 @@ public class QInstance + /******************************************************************************* + ** + *******************************************************************************/ + public void addAutomationProvider(QAutomationProviderMetaData automationProvider) + { + this.addAutomationProvider(automationProvider.getName(), automationProvider); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addAutomationProvider(String name, QAutomationProviderMetaData automationProvider) + { + if(this.automationProviders.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second automationProvider with name: " + name)); + } + this.automationProviders.put(name, automationProvider); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QAutomationProviderMetaData getAutomationProvider(String name) + { + return (this.automationProviders.get(name)); + } + + + + /******************************************************************************* + ** Getter for automationProviders + ** + *******************************************************************************/ + public Map getAutomationProviders() + { + return automationProviders; + } + + + + /******************************************************************************* + ** Setter for automationProviders + ** + *******************************************************************************/ + public void setAutomationProviders(Map automationProviders) + { + this.automationProviders = automationProviders; + } + + + /******************************************************************************* ** Getter for hasBeenValidated ** @@ -426,6 +473,17 @@ public class QInstance + /******************************************************************************* + ** Setter for hasBeenValidated + ** + *******************************************************************************/ + public void setHasBeenValidated(QInstanceValidationKey key) + { + this.hasBeenValidated = true; + } + + + /******************************************************************************* ** Getter for authentication ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/PollingAutomationProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/PollingAutomationProviderMetaData.java new file mode 100644 index 00000000..e5bb0283 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/PollingAutomationProviderMetaData.java @@ -0,0 +1,43 @@ +package com.kingsrook.qqq.backend.core.model.metadata.automation; + + +/******************************************************************************* + ** Metadata specifically for the polling automation provider. + *******************************************************************************/ +public class PollingAutomationProviderMetaData extends QAutomationProviderMetaData +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public PollingAutomationProviderMetaData() + { + super(); + setType(QAutomationProviderType.POLLING); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public PollingAutomationProviderMetaData withName(String name) + { + setName(name); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public PollingAutomationProviderMetaData withType(QAutomationProviderType type) + { + setType(type); + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java new file mode 100644 index 00000000..ae8c8f00 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java @@ -0,0 +1,77 @@ +package com.kingsrook.qqq.backend.core.model.metadata.automation; + + +/******************************************************************************* + ** Meta-data definition of a qqq service to drive record automations. + *******************************************************************************/ +public class QAutomationProviderMetaData +{ + private String name; + private QAutomationProviderType type; + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Setter for name + ** + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public QAutomationProviderMetaData withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public QAutomationProviderType getType() + { + return type; + } + + + + /******************************************************************************* + ** Setter for type + ** + *******************************************************************************/ + public void setType(QAutomationProviderType type) + { + this.type = type; + } + + + /******************************************************************************* + ** Fluent setter for type + ** + *******************************************************************************/ + public QAutomationProviderMetaData withType(QAutomationProviderType type) + { + this.type = type; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderType.java new file mode 100644 index 00000000..9a78e1fe --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderType.java @@ -0,0 +1,60 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.automation; + + +/******************************************************************************* + ** Enum to define the possible automation provider types + ** + *******************************************************************************/ +public enum QAutomationProviderType +{ + POLLING("polling"), + SYNCHRONOUS("synchronous"), + ASYNCHRONOUS("asynchronous"), + MQ("mq"), + AMAZON_SQS("sqs"); + + private final String name; + + + + /******************************************************************************* + ** enum constructor + *******************************************************************************/ + QAutomationProviderType(String name) + { + this.name = name; + } + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return this.name; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeReference.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeReference.java index 5a82f004..40cbc3a2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeReference.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeReference.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.code; +import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; @@ -59,6 +60,17 @@ public class QCodeReference + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return "QCodeReference{name='" + name + "'}"; + } + + + /******************************************************************************* ** Constructor that just takes a java class, and infers the other fields. *******************************************************************************/ @@ -75,6 +87,10 @@ public class QCodeReference { this.codeUsage = QCodeUsage.POSSIBLE_VALUE_PROVIDER; } + else if(RecordAutomationHandler.class.isAssignableFrom(javaClass)) + { + this.codeUsage = QCodeUsage.RECORD_AUTOMATION_HANDLER; + } else { throw (new IllegalStateException("Unable to infer code usage type for class: " + javaClass.getName())); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeUsage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeUsage.java index 925c1f55..7fd8fbd1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeUsage.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeUsage.java @@ -30,5 +30,6 @@ public enum QCodeUsage { BACKEND_STEP, // a backend-step in a process CUSTOMIZER, // a function to customize part of a QQQ table's behavior - POSSIBLE_VALUE_PROVIDER // code that drives a custom possibleValueSource + POSSIBLE_VALUE_PROVIDER, // code that drives a custom possibleValueSource + RECORD_AUTOMATION_HANDLER // code that executes record automations } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index 0d1cbcef..318a12bb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; /******************************************************************************* @@ -63,7 +64,8 @@ public class QTableMetaData implements QAppChildMetaData, Serializable private Map fields; - private QTableBackendDetails backendDetails; + private QTableBackendDetails backendDetails; + private QTableAutomationDetails automationDetails; private Map customizers; @@ -410,6 +412,40 @@ public class QTableMetaData implements QAppChildMetaData, Serializable + /******************************************************************************* + ** Getter for automationDetails + ** + *******************************************************************************/ + public QTableAutomationDetails getAutomationDetails() + { + return automationDetails; + } + + + + /******************************************************************************* + ** Setter for automationDetails + ** + *******************************************************************************/ + public void setAutomationDetails(QTableAutomationDetails automationDetails) + { + this.automationDetails = automationDetails; + } + + + + /******************************************************************************* + ** Fluent Setter for automationDetails + ** + *******************************************************************************/ + public QTableMetaData withAutomationDetails(QTableAutomationDetails automationDetails) + { + this.automationDetails = automationDetails; + return (this); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTracking.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTracking.java new file mode 100644 index 00000000..7d504465 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTracking.java @@ -0,0 +1,84 @@ +package com.kingsrook.qqq.backend.core.model.metadata.tables.automation; + + +/******************************************************************************* + ** Table-automation meta-data to define how this table's per-record automation + ** status is tracked. + *******************************************************************************/ +public class AutomationStatusTracking +{ + private AutomationStatusTrackingType type; + + private String fieldName; // used when type is FIELD_IN_TABLE + + // todo - fields for additional types (e.g., 1-1 table, shared-table) + + + + /******************************************************************************* + ** Getter for type + ** + *******************************************************************************/ + public AutomationStatusTrackingType getType() + { + return type; + } + + + + /******************************************************************************* + ** Setter for type + ** + *******************************************************************************/ + public void setType(AutomationStatusTrackingType type) + { + this.type = type; + } + + + + /******************************************************************************* + ** Fluent setter for type + ** + *******************************************************************************/ + public AutomationStatusTracking withType(AutomationStatusTrackingType type) + { + this.type = type; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldName + ** + *******************************************************************************/ + public String getFieldName() + { + return fieldName; + } + + + + /******************************************************************************* + ** Setter for fieldName + ** + *******************************************************************************/ + public void setFieldName(String fieldName) + { + this.fieldName = fieldName; + } + + + + /******************************************************************************* + ** Fluent setter for fieldName + ** + *******************************************************************************/ + public AutomationStatusTracking withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTrackingType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTrackingType.java new file mode 100644 index 00000000..6d67f046 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTrackingType.java @@ -0,0 +1,11 @@ +package com.kingsrook.qqq.backend.core.model.metadata.tables.automation; + + +/******************************************************************************* + ** Enum of possible types of per-table, per-record automation-status tracking. + *******************************************************************************/ +public enum AutomationStatusTrackingType +{ + FIELD_IN_TABLE + // todo - additional types (e.g., 1-1 table, shared-table) +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java new file mode 100644 index 00000000..fadfc06d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java @@ -0,0 +1,133 @@ +package com.kingsrook.qqq.backend.core.model.metadata.tables.automation; + + +import java.util.ArrayList; +import java.util.List; + + +/******************************************************************************* + ** Details about how this table's record automations are set up. + *******************************************************************************/ +public class QTableAutomationDetails +{ + private AutomationStatusTracking statusTracking; + private String providerName; + private List actions; + + + + /******************************************************************************* + ** Getter for statusTracking + ** + *******************************************************************************/ + public AutomationStatusTracking getStatusTracking() + { + return statusTracking; + } + + + + /******************************************************************************* + ** Setter for statusTracking + ** + *******************************************************************************/ + public void setStatusTracking(AutomationStatusTracking statusTracking) + { + this.statusTracking = statusTracking; + } + + + + /******************************************************************************* + ** Fluent setter for statusTracking + ** + *******************************************************************************/ + public QTableAutomationDetails withStatusTracking(AutomationStatusTracking statusTracking) + { + this.statusTracking = statusTracking; + return (this); + } + + + + /******************************************************************************* + ** Getter for providerName + ** + *******************************************************************************/ + public String getProviderName() + { + return providerName; + } + + + + /******************************************************************************* + ** Setter for providerName + ** + *******************************************************************************/ + public void setProviderName(String providerName) + { + this.providerName = providerName; + } + + + + /******************************************************************************* + ** Fluent setter for providerName + ** + *******************************************************************************/ + public QTableAutomationDetails withProviderName(String providerName) + { + this.providerName = providerName; + return (this); + } + + + + /******************************************************************************* + ** Getter for actions + ** + *******************************************************************************/ + public List getActions() + { + return actions; + } + + + + /******************************************************************************* + ** Setter for actions + ** + *******************************************************************************/ + public void setActions(List actions) + { + this.actions = actions; + } + + + + /******************************************************************************* + ** Fluent setter for actions + ** + *******************************************************************************/ + public QTableAutomationDetails withActions(List actions) + { + this.actions = actions; + return (this); + } + + + + /******************************************************************************* + ** Fluently add an action to this table's automations. + *******************************************************************************/ + public QTableAutomationDetails withAction(TableAutomationAction action) + { + if(this.actions == null) + { + this.actions = new ArrayList<>(); + } + this.actions.add(action); + return (this); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TableAutomationAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TableAutomationAction.java new file mode 100644 index 00000000..295e4bdc --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TableAutomationAction.java @@ -0,0 +1,232 @@ +package com.kingsrook.qqq.backend.core.model.metadata.tables.automation; + + +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; + + +/******************************************************************************* + ** Definition of a specific action to run against a table + *******************************************************************************/ +public class TableAutomationAction +{ + private String name; + private TriggerEvent triggerEvent; + private Integer priority = 500; + private QQueryFilter filter; + + //////////////////////////////// + // mutually-exclusive options // + //////////////////////////////// + private QCodeReference codeReference; + private String processName; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return "TableAutomationAction{name='" + name + "'}";} + + + + /******************************************************************************* + ** Getter for name + ** + *******************************************************************************/ + public String getName() + { + return name; + } + + + + /******************************************************************************* + ** Setter for name + ** + *******************************************************************************/ + public void setName(String name) + { + this.name = name; + } + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public TableAutomationAction withName(String name) + { + this.name = name; + return (this); + } + + + + /******************************************************************************* + ** Getter for triggerEvent + ** + *******************************************************************************/ + public TriggerEvent getTriggerEvent() + { + return triggerEvent; + } + + + + /******************************************************************************* + ** Setter for triggerEvent + ** + *******************************************************************************/ + public void setTriggerEvent(TriggerEvent triggerEvent) + { + this.triggerEvent = triggerEvent; + } + + + /******************************************************************************* + ** Fluent setter for triggerEvent + ** + *******************************************************************************/ + public TableAutomationAction withTriggerEvent(TriggerEvent triggerEvent) + { + this.triggerEvent = triggerEvent; + return (this); + } + + + + /******************************************************************************* + ** Getter for priority + ** + *******************************************************************************/ + public Integer getPriority() + { + return priority; + } + + + + /******************************************************************************* + ** Setter for priority + ** + *******************************************************************************/ + public void setPriority(Integer priority) + { + this.priority = priority; + } + + + /******************************************************************************* + ** Fluent setter for priority + ** + *******************************************************************************/ + public TableAutomationAction withPriority(Integer priority) + { + this.priority = priority; + return (this); + } + + + + /******************************************************************************* + ** Getter for filter + ** + *******************************************************************************/ + public QQueryFilter getFilter() + { + return filter; + } + + + + /******************************************************************************* + ** Setter for filter + ** + *******************************************************************************/ + public void setFilter(QQueryFilter filter) + { + this.filter = filter; + } + + + /******************************************************************************* + ** Fluent setter for filter + ** + *******************************************************************************/ + public TableAutomationAction withFilter(QQueryFilter filter) + { + this.filter = filter; + return (this); + } + + + + /******************************************************************************* + ** Getter for codeReference + ** + *******************************************************************************/ + public QCodeReference getCodeReference() + { + return codeReference; + } + + + + /******************************************************************************* + ** Setter for codeReference + ** + *******************************************************************************/ + public void setCodeReference(QCodeReference codeReference) + { + this.codeReference = codeReference; + } + + + /******************************************************************************* + ** Fluent setter for codeReference + ** + *******************************************************************************/ + public TableAutomationAction withCodeReference(QCodeReference codeReference) + { + this.codeReference = codeReference; + return (this); + } + + + + /******************************************************************************* + ** Getter for processName + ** + *******************************************************************************/ + public String getProcessName() + { + return processName; + } + + + + /******************************************************************************* + ** Setter for processName + ** + *******************************************************************************/ + public void setProcessName(String processName) + { + this.processName = processName; + } + + + /******************************************************************************* + ** Fluent setter for processName + ** + *******************************************************************************/ + public TableAutomationAction withProcessName(String processName) + { + this.processName = processName; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TriggerEvent.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TriggerEvent.java new file mode 100644 index 00000000..4b64ebe2 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TriggerEvent.java @@ -0,0 +1,12 @@ +package com.kingsrook.qqq.backend.core.model.metadata.tables.automation; + + +/******************************************************************************* + ** Possible events that can trigger a record automation. + *******************************************************************************/ +public enum TriggerEvent +{ + POST_INSERT, + POST_UPDATE, + PRE_DELETE +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java new file mode 100644 index 00000000..cfda8981 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java @@ -0,0 +1,233 @@ +package com.kingsrook.qqq.backend.core.actions.automation.polling; + + +import java.time.LocalDate; +import java.time.Month; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for PollingAutomationExecutor + *******************************************************************************/ +class PollingAutomationExecutorTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testInsert() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + + ///////////////////////////////////////////////////////////////////////////// + // insert 2 people - one who should be updated by the check-age automation // + ///////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("firstName", "John").withValue("birthDate", LocalDate.of(1970, Month.JANUARY, 1)), + new QRecord().withValue("id", 2).withValue("firstName", "Jim").withValue("birthDate", LocalDate.now().minusDays(30)) + )); + new InsertAction().execute(insertInput); + + //////////////////////////////////////////////// + // have the polling executor run "for awhile" // + //////////////////////////////////////////////// + runPollingAutomationExecutorForAwhile(qInstance); + + ///////////////////////////////////////////////// + // query for the records - assert their status // + ///////////////////////////////////////////////// + QueryInput queryInput = new QueryInput(qInstance); + queryInput.setSession(new QSession()); + queryInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size()); + + Optional optionalPerson1 = queryOutput.getRecords().stream().filter(r -> r.getValueInteger("id") == 1).findFirst(); + assertThat(optionalPerson1).isPresent(); + QRecord person1 = optionalPerson1.get(); + assertThat(person1.getValueString("firstName")).isEqualTo("John"); + assertThat(person1.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName())).isEqualTo(AutomationStatus.OK.getId()); + + Optional optionalPerson2 = queryOutput.getRecords().stream().filter(r -> r.getValueInteger("id") == 2).findFirst(); + assertThat(optionalPerson2).isPresent(); + QRecord person2 = optionalPerson2.get(); + assertThat(person2.getValueString("firstName")).isEqualTo("Jim" + TestUtils.CheckAge.SUFFIX_FOR_MINORS); + assertThat(person2.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName())).isEqualTo(AutomationStatus.OK.getId()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUpdate() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + + /////////////////////////////////////////////////////////////////////////////// + // insert 2 people - one who should be logged by logger-on-update automation // + /////////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Tim"), + new QRecord().withValue("id", 2).withValue("firstName", "Darin") + )); + new InsertAction().execute(insertInput); + + //////////////////////////////////////////////// + // have the polling executor run "for awhile" // + //////////////////////////////////////////////// + runPollingAutomationExecutorForAwhile(qInstance); + + ////////////////////////////////////////////////// + // assert that the update-automation didn't run // + ////////////////////////////////////////////////// + assertThat(TestUtils.LogPersonUpdate.updatedIds).isNullOrEmpty(); + + UpdateInput updateInput = new UpdateInput(qInstance); + updateInput.setSession(new QSession()); + updateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + updateInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("lastName", "now with a LastName"), + new QRecord().withValue("id", 2).withValue("lastName", "now with a LastName") + )); + new UpdateAction().execute(updateInput); + + //////////////////////////////////////////////// + // have the polling executor run "for awhile" // + //////////////////////////////////////////////// + runPollingAutomationExecutorForAwhile(qInstance); + + /////////////////////////////////////////////////// + // assert that the update-automation DID run now // + /////////////////////////////////////////////////// + assertThat(TestUtils.LogPersonUpdate.updatedIds).contains(2); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSessionSupplier() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + + ////////////////////////////////////////////////////////////////////// + // make the person-memory table's insert-action run a class in here // + ////////////////////////////////////////////////////////////////////// + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .getAutomationDetails().getActions().get(0) + .setCodeReference(new QCodeReference(CaptureSessionIdAutomationHandler.class)); + + ///////////////////// + // insert a person // + ///////////////////// + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Tim") + )); + new InsertAction().execute(insertInput); + + String uuid = UUID.randomUUID().toString(); + QSession session = new QSession(); + session.setIdReference(uuid); + PollingAutomationExecutor.getInstance().setSessionSupplier(() -> session); + + //////////////////////////////////////////////// + // have the polling executor run "for awhile" // + //////////////////////////////////////////////// + runPollingAutomationExecutorForAwhile(qInstance); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // assert that the uuid we put in our session was present in the CaptureSessionIdAutomationHandler // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + assertThat(CaptureSessionIdAutomationHandler.sessionId).isEqualTo(uuid); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class CaptureSessionIdAutomationHandler extends RecordAutomationHandler + { + static String sessionId; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void execute(RecordAutomationInput recordAutomationInput) throws QException + { + sessionId = recordAutomationInput.getSession().getIdReference(); + } + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runPollingAutomationExecutorForAwhile(QInstance qInstance) + { + PollingAutomationExecutor pollingAutomationExecutor = PollingAutomationExecutor.getInstance(); + pollingAutomationExecutor.setInitialDelayMillis(0); + pollingAutomationExecutor.setDelayMillis(100); + pollingAutomationExecutor.start(qInstance, TestUtils.POLLING_AUTOMATION); + SleepUtils.sleep(1, TimeUnit.SECONDS); + pollingAutomationExecutor.stop(); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index ee2464a9..b2b3d95e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -30,6 +30,9 @@ import java.util.function.Consumer; import java.util.function.Function; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; @@ -43,6 +46,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; @@ -162,11 +166,13 @@ class QInstanceValidatorTest qInstance.getBackend("default").setName("notDefault"); qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setName("notGreetPeople"); qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_STATE).setName("notStates"); + qInstance.getAutomationProvider(TestUtils.POLLING_AUTOMATION).setName("notPolling"); }, "Inconsistent naming for table", "Inconsistent naming for backend", "Inconsistent naming for process", - "Inconsistent naming for possibleValueSource" + "Inconsistent naming for possibleValueSource", + "Inconsistent naming for automationProvider" ); } @@ -296,7 +302,7 @@ class QInstanceValidatorTest "Instance of CodeReference could not be created"); assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").withCustomizer(TableCustomizers.POST_QUERY_RECORD.getRole(), new QCodeReference(CustomizerThatIsNotAFunction.class, QCodeUsage.CUSTOMIZER)), - "CodeReference could not be casted"); + "CodeReference is not of the expected type"); assertValidationFailureReasons((qInstance) -> qInstance.getTable("person").withCustomizer(TableCustomizers.POST_QUERY_RECORD.getRole(), new QCodeReference(CustomizerFunctionWithIncorrectTypeParameters.class, QCodeUsage.CUSTOMIZER)), "Error validating customizer type parameters"); @@ -649,6 +655,230 @@ class QInstanceValidatorTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testAutomationProviderType() + { + assertValidationFailureReasons((qInstance) -> qInstance.getAutomationProvider(TestUtils.POLLING_AUTOMATION).setType(null), + "Missing type for automationProvider"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableAutomationProviderName() + { + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().setProviderName(null), + "is missing a providerName"); + + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().setProviderName(""), + "is missing a providerName"); + + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().setProviderName("notARealProvider"), + "unrecognized providerName"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableAutomationStatusTracking() + { + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().setStatusTracking(null), + "do not have statusTracking"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableAutomationStatusTrackingType() + { + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().getStatusTracking().setType(null), + "statusTracking is missing a type"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableAutomationStatusTrackingFieldName() + { + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().getStatusTracking().setFieldName(null), + "missing its fieldName"); + + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().getStatusTracking().setFieldName(""), + "missing its fieldName"); + + ////////////////////////////////////////////////// + // todo - make sure it's a field in the table?? // + ////////////////////////////////////////////////// + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableAutomationActionsNames() + { + assertValidationFailureReasons((qInstance) -> getAction0(qInstance).setName(null), + "action missing a name"); + + assertValidationFailureReasons((qInstance) -> getAction0(qInstance).setName(""), + "action missing a name"); + + assertValidationFailureReasons((qInstance) -> + { + List actions = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().getActions(); + actions.add(actions.get(0)); + }, + "more than one action named"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableAutomationActionTriggerEvent() + { + assertValidationFailureReasons((qInstance) -> getAction0(qInstance).setTriggerEvent(null), + "missing a triggerEvent"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableAutomationActionCodeReference() + { + assertValidationFailureReasons((qInstance) -> getAction0(qInstance).setCodeReference(new QCodeReference()), + "missing a code reference name", "missing a code type"); + + assertValidationFailureReasons((qInstance) -> getAction0(qInstance).setCodeReference(new QCodeReference(TestUtils.CustomPossibleValueSource.class)), + "is not of the expected type"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableAutomationActionProcessName() + { + assertValidationFailureReasons((qInstance) -> + { + TableAutomationAction action = getAction0(qInstance); + action.setCodeReference(null); + action.setProcessName("notAProcess"); + }, + "unrecognized processName"); + + assertValidationSuccess((qInstance) -> + { + qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + TableAutomationAction action = getAction0(qInstance); + action.setCodeReference(null); + action.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE); + }); + + assertValidationFailureReasons((qInstance) -> + { + TableAutomationAction action = getAction0(qInstance); + action.setCodeReference(null); + action.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE); + }, + "different table"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableAutomationActionCodeReferenceAndProcessName() + { + assertValidationFailureReasons((qInstance) -> + { + TableAutomationAction action = getAction0(qInstance); + action.setCodeReference(null); + action.setProcessName(null); + }, + "missing both"); + + assertValidationFailureReasons((qInstance) -> + { + qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + TableAutomationAction action = getAction0(qInstance); + action.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE); + }, + "has both"); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTableAutomationActionFilter() + { + assertValidationFailureReasons((qInstance) -> + { + TableAutomationAction action = getAction0(qInstance); + action.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria()) + ); + }, + "without a field name", "without an operator"); + + assertValidationFailureReasons((qInstance) -> + { + TableAutomationAction action = getAction0(qInstance); + action.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("notAField", QCriteriaOperator.EQUALS, Collections.emptyList())) + ); + }, + "unrecognized field"); + + assertValidationSuccess((qInstance) -> + { + TableAutomationAction action = getAction0(qInstance); + action.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(1701))) + ); + }); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private TableAutomationAction getAction0(QInstance qInstance) + { + return qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().getActions().get(0); + } + + + /******************************************************************************* ** Run a little setup code on a qInstance; then validate it, and assert that it ** failed validation with reasons that match the supplied vararg-reasons (but allow @@ -690,7 +920,8 @@ class QInstanceValidatorTest if(!allowExtraReasons) { int noOfReasons = e.getReasons() == null ? 0 : e.getReasons().size(); - assertEquals(reasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", reasons) + "\nActual reasons: " + e.getReasons()); + assertEquals(reasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", reasons) + + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", e.getReasons()) : "--")); } for(String reason : reasons) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 9058bc4a..5c358fbe 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -23,26 +23,39 @@ package com.kingsrook.qqq.backend.core.utils; import java.io.Serializable; +import java.time.LocalDate; +import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.GetAgeStatistics; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.automation.PollingAutomationProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; @@ -52,7 +65,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; @@ -61,6 +79,8 @@ import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockB import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -69,6 +89,8 @@ import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackend *******************************************************************************/ public class TestUtils { + private static final Logger LOG = LogManager.getLogger(TestUtils.class); + public static final String DEFAULT_BACKEND_NAME = "default"; public static final String MEMORY_BACKEND_NAME = "memory"; @@ -83,11 +105,15 @@ public class TestUtils public static final String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive"; public static final String PROCESS_NAME_ADD_TO_PEOPLES_AGE = "addToPeoplesAge"; public static final String TABLE_NAME_PERSON_FILE = "personFile"; + public static final String TABLE_NAME_PERSON_MEMORY = "personMemory"; public static final String TABLE_NAME_ID_AND_NAME_ONLY = "idAndNameOnly"; - public static final String POSSIBLE_VALUE_SOURCE_STATE = "state"; // enum-type - public static final String POSSIBLE_VALUE_SOURCE_SHAPE = "shape"; // table-type - public static final String POSSIBLE_VALUE_SOURCE_CUSTOM = "custom"; // custom-type + public static final String POSSIBLE_VALUE_SOURCE_STATE = "state"; // enum-type + public static final String POSSIBLE_VALUE_SOURCE_SHAPE = "shape"; // table-type + public static final String POSSIBLE_VALUE_SOURCE_CUSTOM = "custom"; // custom-type + public static final String POSSIBLE_VALUE_SOURCE_AUTOMATION_STATUS = "automationStatus"; + + public static final String POLLING_AUTOMATION = "polling"; @@ -104,9 +130,11 @@ public class TestUtils qInstance.addTable(defineTablePerson()); qInstance.addTable(definePersonFileTable()); + qInstance.addTable(definePersonMemoryTable()); qInstance.addTable(defineTableIdAndNameOnly()); qInstance.addTable(defineTableShape()); + qInstance.addPossibleValueSource(defineAutomationStatusPossibleValueSource()); qInstance.addPossibleValueSource(defineStatesPossibleValueSource()); qInstance.addPossibleValueSource(defineShapePossibleValueSource()); qInstance.addPossibleValueSource(defineCustomPossibleValueSource()); @@ -117,6 +145,8 @@ public class TestUtils qInstance.addProcess(new BasicETLProcess().defineProcessMetaData()); qInstance.addProcess(new StreamedETLProcess().defineProcessMetaData()); + qInstance.addAutomationProvider(definePollingAutomationProvider()); + defineApps(qInstance); return (qInstance); @@ -124,6 +154,18 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + private static QAutomationProviderMetaData definePollingAutomationProvider() + { + return (new PollingAutomationProviderMetaData() + .withName(POLLING_AUTOMATION) + ); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -148,6 +190,21 @@ public class TestUtils + /******************************************************************************* + ** Define the "automationStatus" possible value source used in standard tests + ** + *******************************************************************************/ + private static QPossibleValueSource defineAutomationStatusPossibleValueSource() + { + return new QPossibleValueSource() + .withName(POSSIBLE_VALUE_SOURCE_AUTOMATION_STATUS) + .withType(QPossibleValueSourceType.ENUM) + .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY) + .withValuesFromEnum(AutomationStatus.values()); + } + + + /******************************************************************************* ** Define the "states" possible value source used in standard tests ** @@ -252,6 +309,30 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static QFieldMetaData standardQqqAutomationStatusField() + { + return (new QFieldMetaData("qqqAutomationStatus", QFieldType.INTEGER).withPossibleValueSourceName(POSSIBLE_VALUE_SOURCE_AUTOMATION_STATUS)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QTableAutomationDetails defineStandardAutomationDetails() + { + return (new QTableAutomationDetails() + .withProviderName(POLLING_AUTOMATION) + .withStatusTracking(new AutomationStatusTracking() + .withType(AutomationStatusTrackingType.FIELD_IN_TABLE) + .withFieldName("qqqAutomationStatus"))); + } + + + /******************************************************************************* ** Define the 'shape' table used in standard tests. *******************************************************************************/ @@ -289,6 +370,101 @@ public class TestUtils + /******************************************************************************* + ** Define a 3nd version of the 'person' table, backed by the in-memory backend + *******************************************************************************/ + public static QTableMetaData definePersonMemoryTable() + { + return (new QTableMetaData() + .withName(TABLE_NAME_PERSON_MEMORY) + .withBackendName(MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withFields(TestUtils.defineTablePerson().getFields())) + + .withField(standardQqqAutomationStatusField()) + .withAutomationDetails(defineStandardAutomationDetails() + .withAction(new TableAutomationAction() + .withName("checkAgeOnInsert") + .withTriggerEvent(TriggerEvent.POST_INSERT) + .withCodeReference(new QCodeReference(CheckAge.class)) + ) + .withAction(new TableAutomationAction() + .withName("logOnUpdatePerFilter") + .withTriggerEvent(TriggerEvent.POST_UPDATE) + .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Darin")))) + .withCodeReference(new QCodeReference(LogPersonUpdate.class)) + ) + ); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class CheckAge extends RecordAutomationHandler + { + public static String SUFFIX_FOR_MINORS = " (a minor)"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void execute(RecordAutomationInput recordAutomationInput) throws QException + { + LocalDate limitDate = LocalDate.now().minusYears(18); + List recordsToUpdate = new ArrayList<>(); + for(QRecord record : recordAutomationInput.getRecordList()) + { + LocalDate birthDate = record.getValueLocalDate("birthDate"); + if(birthDate != null && birthDate.isAfter(limitDate)) + { + LOG.info("Person [" + record.getValueInteger("id") + "] is a minor - updating their firstName to state such."); + recordsToUpdate.add(new QRecord() + .withValue("id", record.getValue("id")) + .withValue("firstName", record.getValueString("firstName") + SUFFIX_FOR_MINORS) + ); + } + } + + if(!recordsToUpdate.isEmpty()) + { + UpdateInput updateInput = new UpdateInput(recordAutomationInput.getInstance()); + updateInput.setSession(recordAutomationInput.getSession()); + updateInput.setTableName(recordAutomationInput.getTableName()); + updateInput.setRecords(recordsToUpdate); + new UpdateAction().execute(updateInput); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class LogPersonUpdate extends RecordAutomationHandler + { + public static List updatedIds = new ArrayList<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void execute(RecordAutomationInput recordAutomationInput) throws QException + { + for(QRecord record : recordAutomationInput.getRecordList()) + { + updatedIds.add(record.getValueInteger("id")); + LOG.info("Person [" + record.getValueInteger("id") + ":" + record.getValueString("firstName") + "] has been updated."); + } + } + } + + + /******************************************************************************* ** Define simple table with just an id and name *******************************************************************************/ From 64e801747fba540b1b7d2a6551d5e9e0485ba55e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 1 Sep 2022 15:50:33 -0500 Subject: [PATCH 20/28] QQQ-40 getting closer to production-ready on automations --- .../core/actions/async/AsyncJobManager.java | 4 +- .../actions/async/AsyncRecordPipeLoop.java | 16 +- .../RecordAutomationStatusUpdater.java | 54 +++- .../polling/PollingAutomationRunner.java | 225 +++++++++++----- .../core/actions/reporting/RecordPipe.java | 2 +- .../qqq/backend/core/model/data/QRecord.java | 11 + .../metadata/processes/QProcessMetaData.java | 30 ++- .../memory/MemoryBackendModule.java | 4 + .../memory/MemoryRecordStore.java | 251 +++++++++++++++--- .../StreamedETLPreviewStep.java | 28 ++ .../src/main/resources/log4j2.xml | 4 +- .../PollingAutomationExecutorTest.java | 60 +---- .../polling/PollingAutomationRunnerTest.java | 226 ++++++++++++++++ .../actions/reporting/ReportActionTest.java | 2 +- .../memory/MemoryBackendModuleTest.java | 98 +++++++ .../qqq/backend/core/utils/TestUtils.java | 94 ++++++- 16 files changed, 933 insertions(+), 176 deletions(-) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunnerTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java index fbf95f07..041629e3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java @@ -135,11 +135,11 @@ public class AsyncJobManager Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8)); try { - LOG.info("Starting job " + uuidAndTypeStateKey.getUuid()); + LOG.debug("Starting job " + uuidAndTypeStateKey.getUuid()); T result = asyncJob.run(new AsyncJobCallback(uuidAndTypeStateKey.getUuid(), asyncJobStatus)); asyncJobStatus.setState(AsyncJobState.COMPLETE); getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus); - LOG.info("Completed job " + uuidAndTypeStateKey.getUuid()); + LOG.debug("Completed job " + uuidAndTypeStateKey.getUuid()); return (result); } catch(Exception e) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java index a15948f3..d41782bd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java @@ -48,7 +48,7 @@ public class AsyncRecordPipeLoop /////////////////////////////////////////////////// AsyncJobManager asyncJobManager = new AsyncJobManager(); String jobUUID = asyncJobManager.startJob(jobName, supplier::apply); - LOG.info("Started supplier job [" + jobUUID + "] for record pipe."); + LOG.debug("Started supplier job [" + jobUUID + "] for record pipe."); AsyncJobState jobState = AsyncJobState.RUNNING; AsyncJobStatus asyncJobStatus = null; @@ -66,7 +66,7 @@ public class AsyncRecordPipeLoop // if the pipe is too empty, sleep to let the producer work. // // todo - smarter sleep? like get notified vs. sleep? // /////////////////////////////////////////////////////////////// - LOG.debug("Too few records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work"); + LOG.trace("Too few records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work"); SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS); nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS); @@ -85,7 +85,7 @@ public class AsyncRecordPipeLoop nextSleepMillis = INIT_SLEEP_MS; recordCount += consumer.get(); - LOG.info(String.format("Processed %,d records so far", recordCount)); + LOG.debug(String.format("Processed %,d records so far", recordCount)); if(recordLimit != null && recordCount >= recordLimit) { @@ -117,7 +117,7 @@ public class AsyncRecordPipeLoop jobState = asyncJobStatus.getState(); } - LOG.info("Job [" + jobUUID + "] completed with status: " + asyncJobStatus); + LOG.debug("Job [" + jobUUID + "][" + jobName + "] completed with status: " + asyncJobStatus); /////////////////////////////////// // propagate errors from the job // @@ -133,8 +133,12 @@ public class AsyncRecordPipeLoop recordCount += consumer.get(); long endTime = System.currentTimeMillis(); - LOG.info(String.format("Processed %,d records", recordCount) - + String.format(" at end of job in %,d ms (%.2f records/second).", (endTime - jobStartTime), 1000d * (recordCount / (.001d + (endTime - jobStartTime))))); + + if(recordCount > 0) + { + LOG.info(String.format("Processed %,d records", recordCount) + + String.format(" at end of job [%s] in %,d ms (%.2f records/second).", jobName, (endTime - jobStartTime), 1000d * (recordCount / (.001d + (endTime - jobStartTime))))); + } return (recordCount); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java index 8728f4a6..d31b7297 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java @@ -1,7 +1,9 @@ package com.kingsrook.qqq.backend.core.actions.automation; +import java.util.ArrayList; import java.util.List; +import java.util.Objects; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; @@ -10,6 +12,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import org.apache.commons.lang.NotImplementedException; @@ -37,7 +41,12 @@ public class RecordAutomationStatusUpdater return (false); } - if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS) || automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS)) + if(canWeSkipPendingAndGoToOkay(table, automationStatus)) + { + automationStatus = AutomationStatus.OK; + } + + if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS)) { Exception e = new Exception(); for(StackTraceElement stackTraceElement : e.getStackTrace()) @@ -45,7 +54,7 @@ public class RecordAutomationStatusUpdater String className = stackTraceElement.getClassName(); if(className.contains("com.kingsrook.qqq.backend.core.actions.automation") && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test")) { - LOG.info("Avoiding re-setting automation status to PENDING while running an automation"); + LOG.debug("Avoiding re-setting automation status to PENDING_UPDATE while running an automation"); return (false); } } @@ -66,6 +75,35 @@ public class RecordAutomationStatusUpdater + /******************************************************************************* + ** If a table has no automation actions defined for Insert (or Update), and we're + ** being asked to set status to PENDING_INSERT (or PENDING_UPDATE), then just + ** move the status straight to OK. + *******************************************************************************/ + private static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus) + { + List tableActions = Objects.requireNonNullElse(table.getAutomationDetails().getActions(), new ArrayList<>()); + + if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS)) + { + if(tableActions.stream().noneMatch(a -> TriggerEvent.POST_INSERT.equals(a.getTriggerEvent()))) + { + return (true); + } + } + else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS)) + { + if(tableActions.stream().noneMatch(a -> TriggerEvent.POST_UPDATE.equals(a.getTriggerEvent()))) + { + return (true); + } + } + + return (false); + } + + + /******************************************************************************* ** for a list of records, update their automation status and actually Update the ** backend as well. @@ -81,7 +119,17 @@ public class RecordAutomationStatusUpdater UpdateInput updateInput = new UpdateInput(instance); updateInput.setSession(session); updateInput.setTableName(table.getName()); - updateInput.setRecords(records); + + ///////////////////////////////////////////////////////////////////////////////////// + // build records with just their pkey & status field for this update, to avoid // + // changing other values (relies on assumption of Patch semantics in UpdateAction) // + ///////////////////////////////////////////////////////////////////////////////////// + updateInput.setRecords(records.stream().map(r -> new QRecord() + .withTableName(r.getTableName()) + .withValue(table.getPrimaryKeyField(), r.getValue(table.getPrimaryKeyField())) + .withValue(automationDetails.getStatusTracking().getFieldName(), r.getValue(automationDetails.getStatusTracking().getFieldName()))).toList()); + updateInput.setAreAllValuesBeingUpdatedTheSame(true); + new UpdateAction().execute(updateInput); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java index 46d69208..06f31946 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java @@ -1,33 +1,41 @@ package com.kingsrook.qqq.backend.core.actions.automation.polling; +import java.io.Serializable; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Supplier; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; -import org.apache.commons.lang.NotImplementedException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -40,8 +48,8 @@ class PollingAutomationRunner implements Runnable { private static final Logger LOG = LogManager.getLogger(PollingAutomationRunner.class); - private QInstance instance; - private String providerName; + private QInstance instance; + private String providerName; private Supplier sessionSupplier; private List managedTables = new ArrayList<>(); @@ -130,7 +138,7 @@ class PollingAutomationRunner implements Runnable /******************************************************************************* - ** + ** Query for and process records that have a PENDING status on a given table. *******************************************************************************/ private void processTable(QTableMetaData table) throws QException { @@ -142,7 +150,7 @@ class PollingAutomationRunner implements Runnable /******************************************************************************* - ** + ** Query for and process records that have a PENDING_INSERT or PENDING_UPDATE status on a given table. *******************************************************************************/ private void processTableInsertOrUpdate(QTableMetaData table, QSession session, boolean isInsert) throws QException { @@ -153,88 +161,185 @@ class PollingAutomationRunner implements Runnable return; } - LOG.info(" Query for records " + automationStatus + " in " + table); + LOG.debug(" Query for records " + automationStatus + " in " + table); - QueryInput queryInput = new QueryInput(instance); - queryInput.setSession(session); // todo - where the heck can we get this from?? - queryInput.setTableName(table.getName()); + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // run an async-pipe loop - that will query for records in PENDING - put them in a pipe - then apply actions to them // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + RecordPipe recordPipe = new RecordPipe(); + AsyncRecordPipeLoop asyncRecordPipeLoop = new AsyncRecordPipeLoop(); + asyncRecordPipeLoop.run("PollingAutomationRunner>Query>" + (isInsert ? "insert" : "update"), null, recordPipe, (status) -> + { + QueryInput queryInput = new QueryInput(instance); + queryInput.setSession(session); + queryInput.setTableName(table.getName()); + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(table.getAutomationDetails().getStatusTracking().getFieldName(), QCriteriaOperator.IN, List.of(automationStatus.getId())))); + queryInput.setRecordPipe(recordPipe); + return (new QueryAction().execute(queryInput)); + }, () -> + { + List records = recordPipe.consumeAvailableRecords(); + applyActionsToRecords(session, table, records, actions, isInsert); + return (records.size()); + } + ); + } + + + /******************************************************************************* + ** For a set of records that were found to be in a PENDING state - run all the + ** table's actions against them. + *******************************************************************************/ + private void applyActionsToRecords(QSession session, QTableMetaData table, List records, List actions, boolean isInsert) throws QException + { + if(CollectionUtils.nullSafeIsEmpty(records)) + { + return; + } + + /////////////////////////////////////////////////// + // mark the records as RUNNING their automations // + /////////////////////////////////////////////////// + RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, isInsert ? AutomationStatus.RUNNING_INSERT_AUTOMATIONS : AutomationStatus.RUNNING_UPDATE_AUTOMATIONS); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // foreach action - run it against the records (but only if they match the action's filter, if there is one) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + boolean anyActionsFailed = false; for(TableAutomationAction action : actions) { - QQueryFilter filter = action.getFilter(); - if(filter == null) + try { - filter = new QQueryFilter(); + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // note - this method - will re-query the objects, so we should have confidence that their data is fresh... // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List matchingQRecords = getRecordsMatchingActionFilter(session, table, records, action); + LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action); + if(CollectionUtils.nullSafeHasContents(matchingQRecords)) + { + LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action); + applyActionToMatchingRecords(session, table, matchingQRecords, action); + } } - - filter.addCriteria(new QFilterCriteria(table.getAutomationDetails().getStatusTracking().getFieldName(), QCriteriaOperator.IN, List.of(automationStatus.getId()))); - queryInput.setFilter(filter); - - QueryOutput queryOutput = new QueryAction().execute(queryInput); - - // todo - pipe this query!! - - if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords())) + catch(Exception e) { - LOG.info(" Processing " + queryOutput.getRecords().size() + " records in " + table + " for action " + action); - processRecords(table, actions, queryOutput.getRecords(), session, isInsert); + LOG.warn("Caught exception processing records on " + table + " for action " + action, e); + anyActionsFailed = true; } } + + //////////////////////////////////////// + // update status on all these records // + //////////////////////////////////////// + if(anyActionsFailed) + { + RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.FAILED_UPDATE_AUTOMATIONS); + } + else + { + RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.OK); + } } /******************************************************************************* + ** For a given action, and a list of records - return a new list, of the ones + ** which match the action's filter (if there is one - if not, then all match). ** + ** Note that this WILL re-query the objects (ALWAYS - even if the action has no filter). + ** This has the nice side effect of always giving fresh/updated records, despite having + ** some cost. + ** + ** At one point, we considered just applying the filter using java-comparisons, + ** but that will almost certainly give potentially different results than a true + ** backend - e.g., just consider if the DB is case-sensitive for strings... *******************************************************************************/ - private void processRecords(QTableMetaData table, List actions, List records, QSession session, boolean isInsert) throws QException + private List getRecordsMatchingActionFilter(QSession session, QTableMetaData table, List records, TableAutomationAction action) throws QException { - try - { - updateRecordAutomationStatus(table, session, records, isInsert ? AutomationStatus.RUNNING_INSERT_AUTOMATIONS : AutomationStatus.RUNNING_UPDATE_AUTOMATIONS); + QueryInput queryInput = new QueryInput(instance); + queryInput.setSession(session); + queryInput.setTableName(table.getName()); - for(TableAutomationAction action : actions) + QQueryFilter filter = new QQueryFilter(); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // copy filter criteria from the action's filter to a new filter that we'll run here. // + // Critically - don't modify the filter object on the action! as that object has a long lifespan. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + if(action.getFilter() != null) + { + if(action.getFilter().getCriteria() != null) { - //////////////////////////////////// - // todo - what, re-query them? :( // - //////////////////////////////////// - if(StringUtils.hasContent(action.getProcessName())) - { - ////////////////////////////////////////////////////////////////////////////////////////////// - // todo - uh, how to make these records the input, where an extract step might be involved? // - // should extract step ... see record list and just use it? i think maybe? // - ////////////////////////////////////////////////////////////////////////////////////////////// - throw (new NotImplementedException("processes for automation not yet implemented")); - } - else if(action.getCodeReference() != null) - { - LOG.info(" Executing action: [" + action.getName() + "] as code reference: " + action.getCodeReference()); - RecordAutomationInput input = new RecordAutomationInput(instance); - input.setSession(session); - input.setTableName(table.getName()); - input.setRecordList(records); - - RecordAutomationHandler recordAutomationHandler = QCodeLoader.getRecordAutomationHandler(action); - recordAutomationHandler.execute(input); - } + action.getFilter().getCriteria().forEach(filter::addCriteria); } + if(action.getFilter().getOrderBys() != null) + { + action.getFilter().getOrderBys().forEach(filter::addOrderBy); + } + } - updateRecordAutomationStatus(table, session, records, AutomationStatus.OK); - } - catch(Exception e) - { - updateRecordAutomationStatus(table, session, records, isInsert ? AutomationStatus.FAILED_INSERT_AUTOMATIONS : AutomationStatus.FAILED_UPDATE_AUTOMATIONS); - } + filter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList())); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // always add order-by the primary key, to give more predictable/consistent results // + // todo - in future - if this becomes a source of slowness, make this a config to opt-out? // + ///////////////////////////////////////////////////////////////////////////////////////////// + filter.addOrderBy(new QFilterOrderBy().withFieldName(table.getPrimaryKeyField())); + + queryInput.setFilter(filter); + + return (new QueryAction().execute(queryInput).getRecords()); } /******************************************************************************* - ** + ** Finally, actually run action code against a list of known matching records. *******************************************************************************/ - private void updateRecordAutomationStatus(QTableMetaData table, QSession session, List records, AutomationStatus automationStatus) throws QException + private void applyActionToMatchingRecords(QSession session, QTableMetaData table, List records, TableAutomationAction action) throws Exception { - RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, automationStatus); + if(StringUtils.hasContent(action.getProcessName())) + { + RunProcessInput runProcessInput = new RunProcessInput(instance); + runProcessInput.setSession(session); + runProcessInput.setProcessName(action.getProcessName()); + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // kinda hacky - if we see that this process has an input field of a given name, then put a filter in there to find these records... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QProcessMetaData process = instance.getProcess(action.getProcessName()); + if(process.getInputFields().stream().anyMatch(f -> f.getName().equals(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER))) + { + List recordIds = records.stream().map(r -> r.getValueInteger(table.getPrimaryKeyField())).collect(Collectors.toList()); + QQueryFilter queryFilter = new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds)); + runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, queryFilter); + } + else + { + runProcessInput.setRecords(records); + } + + RunProcessAction runProcessAction = new RunProcessAction(); + RunProcessOutput runProcessOutput = runProcessAction.execute(runProcessInput); + if(runProcessOutput.getException().isPresent()) + { + throw (runProcessOutput.getException().get()); + } + } + else if(action.getCodeReference() != null) + { + LOG.debug(" Executing action: [" + action.getName() + "] as code reference: " + action.getCodeReference()); + RecordAutomationInput input = new RecordAutomationInput(instance); + input.setSession(session); + input.setTableName(table.getName()); + input.setRecordList(records); + + RecordAutomationHandler recordAutomationHandler = QCodeLoader.getRecordAutomationHandler(action); + recordAutomationHandler.execute(input); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java index 87b348b7..548e8869 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java @@ -118,7 +118,7 @@ public class RecordPipe LOG.warn("Giving up adding record to pipe, due to pipe being full for more than {} millis", MAX_SLEEP_LOOP_MILLIS); throw (new IllegalStateException("Giving up adding record to pipe, due to pipe staying full too long.")); } - LOG.debug("Record pipe.add failed (due to full pipe). Blocking."); + LOG.trace("Record pipe.add failed (due to full pipe). Blocking."); SleepUtils.sleep(BLOCKING_SLEEP_MILLIS, TimeUnit.MILLISECONDS); offerResult = queue.offer(record); now = System.currentTimeMillis(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index 29903fc3..4206c97b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -104,6 +104,17 @@ public class QRecord implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return "QRecord{tableName='" + tableName + "',id='" + getValue("id") + "'}"; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index 20b40e02..45247881 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -24,8 +24,10 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; @@ -273,17 +275,25 @@ public class QProcessMetaData implements QAppChildMetaData /******************************************************************************* - ** Get a list of all of the input fields used by all the steps in this process. + ** Get a list of all the *unique* input fields used by all the steps in this process. *******************************************************************************/ @JsonIgnore public List getInputFields() { - List rs = new ArrayList<>(); + Set usedFieldNames = new HashSet<>(); + List rs = new ArrayList<>(); if(steps != null) { for(QStepMetaData step : steps.values()) { - rs.addAll(step.getInputFields()); + for(QFieldMetaData field : step.getInputFields()) + { + if(!usedFieldNames.contains(field.getName())) + { + rs.add(field); + usedFieldNames.add(field.getName()); + } + } } } return (rs); @@ -292,17 +302,25 @@ public class QProcessMetaData implements QAppChildMetaData /******************************************************************************* - ** Get a list of all of the output fields used by all the steps in this process. + ** Get a list of all the *unique* output fields used by all the steps in this process. *******************************************************************************/ @JsonIgnore public List getOutputFields() { - List rs = new ArrayList<>(); + Set usedFieldNames = new HashSet<>(); + List rs = new ArrayList<>(); if(steps != null) { for(QStepMetaData step : steps.values()) { - rs.addAll(step.getOutputFields()); + for(QFieldMetaData field : step.getOutputFields()) + { + if(!usedFieldNames.contains(field.getName())) + { + rs.add(field); + usedFieldNames.add(field.getName()); + } + } } } return (rs); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java index 84852cb9..30346a7f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java @@ -35,6 +35,10 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; ** A simple (probably only valid for testing?) implementation of the QModuleInterface, ** that just stores its records in-memory. ** + ** In general, this class is intended to behave, as much as possible, like an RDBMS. + ** + ** TODO - in future, if we need to - make configs for things like "case-insensitive", + ** and "allow loose typing". *******************************************************************************/ public class MemoryBackendModule implements QBackendModuleInterface { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java index 90eedd28..8dbd9d07 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory; import java.io.Serializable; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -32,12 +33,14 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import org.apache.commons.lang.NotImplementedException; @@ -121,42 +124,7 @@ public class MemoryRecordStore for(QRecord qRecord : tableData.values()) { - boolean recordMatches = true; - if(input.getFilter() != null && input.getFilter().getCriteria() != null) - { - for(QFilterCriteria criterion : input.getFilter().getCriteria()) - { - String fieldName = criterion.getFieldName(); - Serializable value = qRecord.getValue(fieldName); - switch(criterion.getOperator()) - { - case EQUALS: - { - if(!value.equals(criterion.getValues().get(0))) - { - recordMatches = false; - } - break; - } - case IN: - { - if(!criterion.getValues().contains(value)) - { - recordMatches = false; - } - break; - } - default: - { - throw new NotImplementedException("Operator [" + criterion.getOperator() + "] is not yet implemented in the Memory backend."); - } - } - if(!recordMatches) - { - break; - } - } - } + boolean recordMatches = doesRecordMatch(input.getFilter(), qRecord); if(recordMatches) { @@ -169,6 +137,217 @@ public class MemoryRecordStore + /******************************************************************************* + ** + *******************************************************************************/ + private boolean doesRecordMatch(QQueryFilter filter, QRecord qRecord) + { + boolean recordMatches = true; + if(filter != null && filter.getCriteria() != null) + { + for(QFilterCriteria criterion : filter.getCriteria()) + { + String fieldName = criterion.getFieldName(); + Serializable value = qRecord.getValue(fieldName); + + switch(criterion.getOperator()) + { + case EQUALS: + { + recordMatches = testEquals(criterion, value); + break; + } + case NOT_EQUALS: + { + recordMatches = !testEquals(criterion, value); + break; + } + case IN: + { + recordMatches = testIn(criterion, value); + break; + } + case NOT_IN: + { + recordMatches = !testIn(criterion, value); + break; + } + case CONTAINS: + { + recordMatches = testContains(criterion, fieldName, value); + break; + } + case NOT_CONTAINS: + { + recordMatches = !testContains(criterion, fieldName, value); + break; + } + case GREATER_THAN: + { + recordMatches = testGreaterThan(criterion, value); + break; + } + case GREATER_THAN_OR_EQUALS: + { + recordMatches = testGreaterThan(criterion, value) || testEquals(criterion, value); + break; + } + case LESS_THAN: + { + recordMatches = !testGreaterThan(criterion, value) && !testEquals(criterion, value); + break; + } + case LESS_THAN_OR_EQUALS: + { + recordMatches = !testGreaterThan(criterion, value); + break; + } + default: + { + throw new NotImplementedException("Operator [" + criterion.getOperator() + "] is not yet implemented in the Memory backend."); + } + } + if(!recordMatches) + { + break; + } + } + } + return recordMatches; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean testGreaterThan(QFilterCriteria criterion, Serializable value) + { + Serializable criterionValue = criterion.getValues().get(0); + if(criterionValue == null) + { + throw (new IllegalArgumentException("Missing criterion value in query")); + } + + if (value == null) + { + ///////////////////////////////////////////////////////////////////////////////////// + // a database would say 'false' for if a null column is > a value, so do the same. // + ///////////////////////////////////////////////////////////////////////////////////// + return (false); + } + + if(value instanceof LocalDate valueDate && criterionValue instanceof LocalDate criterionValueDate) + { + return (valueDate.isAfter(criterionValueDate)); + } + + if(value instanceof Number valueNumber && criterionValue instanceof Number criterionValueNumber) + { + return (valueNumber.doubleValue() > criterionValueNumber.doubleValue()); + } + + throw (new NotImplementedException("Greater/Less Than comparisons are not (yet?) implemented for the supplied types [" + value.getClass().getSimpleName() + "][" + criterionValue.getClass().getSimpleName() + "]")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean testIn(QFilterCriteria criterion, Serializable value) + { + if(!criterion.getValues().contains(value)) + { + return (false); + } + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean testEquals(QFilterCriteria criterion, Serializable value) + { + if(value == null) + { + return (false); + } + + if(!value.equals(criterion.getValues().get(0))) + { + return (false); + } + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean testContains(QFilterCriteria criterion, String fieldName, Serializable value) + { + String stringValue = getStringFieldValue(value, fieldName, criterion); + String criterionValue = getFirstStringCriterionValue(criterion); + + if(!stringValue.contains(criterionValue)) + { + return (false); + } + + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getFirstStringCriterionValue(QFilterCriteria criteria) + { + if(CollectionUtils.nullSafeIsEmpty(criteria.getValues())) + { + throw new IllegalArgumentException("Missing value for [" + criteria.getOperator() + "] criteria on field [" + criteria.getFieldName() + "]"); + } + Serializable value = criteria.getValues().get(0); + if(value == null) + { + return ""; + } + + if(!(value instanceof String stringValue)) + { + throw new ClassCastException("Value [" + value + "] for criteria [" + criteria.getFieldName() + "] is not a String, which is required for the [" + criteria.getOperator() + "] operator."); + } + + return (stringValue); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getStringFieldValue(Serializable value, String fieldName, QFilterCriteria criterion) + { + if(value == null) + { + return ""; + } + + if(!(value instanceof String stringValue)) + { + throw new ClassCastException("Value [" + value + "] in field [" + fieldName + "] is not a String, which is required for the [" + criterion.getOperator() + "] operator."); + } + + return (stringValue); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java index d16c40b5..59eb69c1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -65,6 +65,15 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe return; } + ///////////////////////////////////////////////////////////////// + // if we're running inside an automation, then skip this step. // + ///////////////////////////////////////////////////////////////// + if(runningWithinAutomation()) + { + LOG.info("Skipping preview step when [" + runBackendStepInput.getProcessName() + "] is running as part of an automation."); + return; + } + /////////////////////////////////////////// // request a count from the extract step // /////////////////////////////////////////// @@ -109,6 +118,25 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe + /******************************************************************************* + ** + *******************************************************************************/ + private boolean runningWithinAutomation() + { + Exception e = new Exception(); + for(StackTraceElement stackTraceElement : e.getStackTrace()) + { + String className = stackTraceElement.getClassName(); + if(className.contains("com.kingsrook.qqq.backend.core.actions.automation")) + { + return (true); + } + } + return false; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/resources/log4j2.xml b/qqq-backend-core/src/main/resources/log4j2.xml index d69b17bc..a16e77af 100644 --- a/qqq-backend-core/src/main/resources/log4j2.xml +++ b/qqq-backend-core/src/main/resources/log4j2.xml @@ -2,11 +2,11 @@ - + - + diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java index cfda8981..a9cbb572 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java @@ -11,12 +11,10 @@ import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; -import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -99,56 +97,6 @@ class PollingAutomationExecutorTest - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testUpdate() throws QException - { - QInstance qInstance = TestUtils.defineInstance(); - - /////////////////////////////////////////////////////////////////////////////// - // insert 2 people - one who should be logged by logger-on-update automation // - /////////////////////////////////////////////////////////////////////////////// - InsertInput insertInput = new InsertInput(qInstance); - insertInput.setSession(new QSession()); - insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); - insertInput.setRecords(List.of( - new QRecord().withValue("id", 1).withValue("firstName", "Tim"), - new QRecord().withValue("id", 2).withValue("firstName", "Darin") - )); - new InsertAction().execute(insertInput); - - //////////////////////////////////////////////// - // have the polling executor run "for awhile" // - //////////////////////////////////////////////// - runPollingAutomationExecutorForAwhile(qInstance); - - ////////////////////////////////////////////////// - // assert that the update-automation didn't run // - ////////////////////////////////////////////////// - assertThat(TestUtils.LogPersonUpdate.updatedIds).isNullOrEmpty(); - - UpdateInput updateInput = new UpdateInput(qInstance); - updateInput.setSession(new QSession()); - updateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); - updateInput.setRecords(List.of( - new QRecord().withValue("id", 1).withValue("lastName", "now with a LastName"), - new QRecord().withValue("id", 2).withValue("lastName", "now with a LastName") - )); - new UpdateAction().execute(updateInput); - - //////////////////////////////////////////////// - // have the polling executor run "for awhile" // - //////////////////////////////////////////////// - runPollingAutomationExecutorForAwhile(qInstance); - - /////////////////////////////////////////////////// - // assert that the update-automation DID run now // - /////////////////////////////////////////////////// - assertThat(TestUtils.LogPersonUpdate.updatedIds).contains(2); - } - /******************************************************************************* @@ -166,14 +114,14 @@ class PollingAutomationExecutorTest .getAutomationDetails().getActions().get(0) .setCodeReference(new QCodeReference(CaptureSessionIdAutomationHandler.class)); - ///////////////////// - // insert a person // - ///////////////////// + //////////////////////////////////////////////////////////// + // insert a person that will trigger the on-insert action // + //////////////////////////////////////////////////////////// InsertInput insertInput = new InsertInput(qInstance); insertInput.setSession(new QSession()); insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); insertInput.setRecords(List.of( - new QRecord().withValue("id", 1).withValue("firstName", "Tim") + new QRecord().withValue("id", 1).withValue("firstName", "Tim").withValue("birthDate", LocalDate.now()) )); new InsertAction().execute(insertInput); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunnerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunnerTest.java new file mode 100644 index 00000000..0b5da000 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunnerTest.java @@ -0,0 +1,226 @@ +package com.kingsrook.qqq.backend.core.actions.automation.polling; + + +import java.time.LocalDate; +import java.time.Month; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for PollingAutomationRunner + *******************************************************************************/ +class PollingAutomationRunnerTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** Test a cycle that does an insert, some automations, then and an update, and more automations. + *******************************************************************************/ + @Test + void testInsertAndUpdate() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, TestUtils.POLLING_AUTOMATION, null); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // insert 2 person records, one who should be both updated by the insert action, and should be logged by logger-on-update automation // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Tim").withValue("birthDate", LocalDate.now()), + new QRecord().withValue("id", 2).withValue("firstName", "Darin") + )); + new InsertAction().execute(insertInput); + assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS); + + ////////////////////////////////////////////////////////////////////////////////////////// + // assert that the update-automation won't run - as no UPDATE has happened on the table // + // even though the insert action does update the records!! // + ////////////////////////////////////////////////////////////////////////////////////////// + pollingAutomationRunner.run(); + assertThat(TestUtils.LogPersonUpdate.updatedIds).isNullOrEmpty(); + assertAllRecordsAutomationStatus(AutomationStatus.OK); + + //////////////////////////////////////////// + // make sure the minor person was updated // + //////////////////////////////////////////// + Optional updatedMinorRecord = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY).stream().filter(r -> r.getValueInteger("id").equals(1)).findFirst(); + assertThat(updatedMinorRecord) + .isPresent() + .get() + .extracting(r -> r.getValueString("firstName")) + .isEqualTo("Tim" + TestUtils.CheckAge.SUFFIX_FOR_MINORS); + + ///////////////////////////////////////////////////////////////////////////////////////// + // run automations again - make sure that there haven't been any updates triggered yet // + ///////////////////////////////////////////////////////////////////////////////////////// + pollingAutomationRunner.run(); + assertThat(TestUtils.LogPersonUpdate.updatedIds).isNullOrEmpty(); + assertAllRecordsAutomationStatus(AutomationStatus.OK); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now do an user-driven update - this SHOULD trigger the update automation next time we run automations. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + UpdateInput updateInput = new UpdateInput(qInstance); + updateInput.setSession(new QSession()); + updateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + updateInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("lastName", "now with a LastName"), + new QRecord().withValue("id", 2).withValue("lastName", "now with a LastName") + )); + new UpdateAction().execute(updateInput); + assertAllRecordsAutomationStatus(AutomationStatus.PENDING_UPDATE_AUTOMATIONS); + + ///////////////////////////////////////////////////////////////////////////////// + // assert that the update-automation DOES run now - and that it only runs once // + // note that it will only run on a sub-set of the records // + ///////////////////////////////////////////////////////////////////////////////// + pollingAutomationRunner.run(); + assertThat(TestUtils.LogPersonUpdate.updatedIds) + .contains(2) + .hasSize(1); + assertAllRecordsAutomationStatus(AutomationStatus.OK); + + ///////////////////////////////////////////////////// + // re-run and assert no further automations happen // + ///////////////////////////////////////////////////// + TestUtils.LogPersonUpdate.updatedIds.clear(); + pollingAutomationRunner.run(); + assertThat(TestUtils.LogPersonUpdate.updatedIds).isNullOrEmpty(); + assertAllRecordsAutomationStatus(AutomationStatus.OK); + } + + + + /******************************************************************************* + ** Test a large-ish number - to demonstrate paging working. + ** + ** Note - this caught an issue during original development, where the QueryFilter + ** attached to the Action was being re-used, w/ new "id IN *" criteria being re-added + ** to it - so, good test. + *******************************************************************************/ + @Test + void testMultiPages() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, TestUtils.POLLING_AUTOMATION, null); + + ////////////////////////////////////////////////////////////////////////////////// + // insert many people - half who should be updated by the AgeChecker automation // + ////////////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + + insertInput.setRecords(new ArrayList<>()); + int SIZE = 2_500; + for(int i = 0; i < SIZE; i++) + { + insertInput.getRecords().add(new QRecord().withValue("firstName", "Tim").withValue("lastName", "Number " + i).withValue("birthDate", LocalDate.now())); + insertInput.getRecords().add(new QRecord().withValue("firstName", "Darin").withValue("lastName", "Number " + i)); + } + + new InsertAction().execute(insertInput); + assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS); + + ///////////////////////// + // run the automations // + ///////////////////////// + pollingAutomationRunner.run(); + assertAllRecordsAutomationStatus(AutomationStatus.OK); + + /////////////////////////////////////////////////////////////////////////// + // make sure that all 'minor' persons were updated (e.g., all the Tim's) // + /////////////////////////////////////////////////////////////////////////// + int updatedMinorsCount = 0; + for(QRecord qRecord : TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)) + { + if(qRecord.getValueString("firstName").startsWith("Tim")) + { + assertEquals("Tim" + TestUtils.CheckAge.SUFFIX_FOR_MINORS, qRecord.getValueString("firstName")); + updatedMinorsCount++; + } + } + + assertEquals(SIZE, updatedMinorsCount, "Expected number of updated records"); + } + + + + /******************************************************************************* + ** Test a cycle that does an insert, some automations, then and an update, and more automations. + *******************************************************************************/ + @Test + void testRunningProcess() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, TestUtils.POLLING_AUTOMATION, null); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // insert 2 person records, one who should be both updated by the insert action, and should be logged by logger-on-update automation // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Tim").withValue("birthDate", LocalDate.of(1886, Month.JUNE, 6)), + new QRecord().withValue("id", 2).withValue("firstName", "Darin").withValue("birthDate", LocalDate.of(1904, Month.APRIL, 4)) + )); + new InsertAction().execute(insertInput); + + pollingAutomationRunner.run(); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // make sure the process ran - which means, it would have updated Tim's birth year to 1900 // + ///////////////////////////////////////////////////////////////////////////////////////////// + Optional updatedMinorRecord = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY).stream().filter(r -> r.getValueInteger("id").equals(1)).findFirst(); + assertThat(updatedMinorRecord) + .isPresent() + .get() + .extracting(r -> r.getValueLocalDate("birthDate").getYear()) + .isEqualTo(1900); + + } + + + /******************************************************************************* + ** + *******************************************************************************/ + private void assertAllRecordsAutomationStatus(AutomationStatus pendingInsertAutomations) throws QException + { + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)) + .isNotEmpty() + .allMatch(r -> pendingInsertAutomations.getId().equals(r.getValue(TestUtils.standardQqqAutomationStatusField().getName()))); + } +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java index 08d0d5ba..aabef749 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java @@ -81,7 +81,7 @@ class ReportActionTest public void testBigger() throws Exception { // int recordCount = 2_000_000; // to really stress locally, use this. - int recordCount = 50_000; + int recordCount = 10_000; String filename = "/tmp/ReportActionTest.csv"; runReport(recordCount, filename, ReportFormat.CSV, false); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java index 9bc5081f..567129b9 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java @@ -22,6 +22,8 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory; +import java.time.LocalDate; +import java.time.Month; import java.util.List; import java.util.function.Function; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; @@ -54,6 +56,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -193,6 +196,101 @@ class MemoryBackendModuleTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueryOperators() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_SHAPE); + QSession session = new QSession(); + + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(session); + insertInput.setTableName(table.getName()); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("name", "Square").withValue("date", LocalDate.of(1980, Month.MAY, 31)), + new QRecord().withValue("id", 2).withValue("name", "Triangle").withValue("date", LocalDate.of(1999, Month.DECEMBER, 31)) + )); + new InsertAction().execute(insertInput); + + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 2))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.IN, List.of(2, 3))).size()); + + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.NOT_IN, List.of(3, 4))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.NOT_IN, List.of(2, 3))).size()); + + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.EQUALS, List.of("Square"))).size()); + assertEquals("Square", queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.EQUALS, List.of("Square"))).get(0).getValue("name")); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("notAFieldSoNull", QCriteriaOperator.EQUALS, List.of("Square"))).size()); + + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.NOT_EQUALS, List.of("notFound"))).size()); + assertEquals("Square", queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.NOT_EQUALS, List.of("Triangle"))).get(0).getValue("name")); + + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.CONTAINS, List.of("ria"))).size()); + assertEquals("Triangle", queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.CONTAINS, List.of("ria"))).get(0).getValue("name")); + + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.NOT_CONTAINS, List.of("notFound"))).size()); + assertEquals("Square", queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.NOT_CONTAINS, List.of("ria"))).get(0).getValue("name")); + + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.CONTAINS, List.of("ria")))); + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.CONTAINS, List.of(1)))); + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.CONTAINS, List.of()))); + + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN, List.of(LocalDate.of(2022, Month.SEPTEMBER, 1)))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN, List.of(LocalDate.of(1990, Month.JANUARY, 1)))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN, List.of(LocalDate.of(1970, Month.JANUARY, 1)))).size()); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, List.of(2))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, List.of(1))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, List.of(0))).size()); + + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(LocalDate.of(2022, Month.SEPTEMBER, 1)))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(LocalDate.of(1990, Month.JANUARY, 1)))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(LocalDate.of(1970, Month.JANUARY, 1)))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(LocalDate.of(1980, Month.MAY, 31)))).size()); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(3))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(2))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(1))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(0))).size()); + + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN, List.of(LocalDate.of(2022, Month.SEPTEMBER, 1)))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN, List.of(LocalDate.of(1990, Month.JANUARY, 1)))).size()); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN, List.of(LocalDate.of(1970, Month.JANUARY, 1)))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN, List.of(3))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN, List.of(2))).size()); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN, List.of(1))).size()); + + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(LocalDate.of(2022, Month.SEPTEMBER, 1)))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(LocalDate.of(1990, Month.JANUARY, 1)))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(LocalDate.of(1980, Month.MAY, 31)))).size()); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(LocalDate.of(1970, Month.JANUARY, 1)))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(3))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(2))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(1))).size()); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(0))).size()); + + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, List.of()))); + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of()))); + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN, List.of()))); + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of()))); + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of("Bob")))); + } + + + + private List queryShapes(QInstance qInstance, QTableMetaData table, QSession session, QFilterCriteria criteria) throws QException + { + QueryInput queryInput = new QueryInput(qInstance); + queryInput.setSession(session); + queryInput.setTableName(table.getName()); + queryInput.setFilter(new QQueryFilter().withCriteria(criteria)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + return queryOutput.getRecords(); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index b0079f5b..cc0baabc 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -24,10 +24,12 @@ package com.kingsrook.qqq.backend.core.utils; import java.io.Serializable; import java.time.LocalDate; +import java.time.Month; import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.GetAgeStatistics; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; @@ -35,6 +37,8 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; @@ -65,10 +69,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -103,6 +107,7 @@ public class TestUtils public static final String PROCESS_NAME_GREET_PEOPLE = "greet"; public static final String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive"; + public static final String PROCESS_NAME_INCREASE_BIRTHDATE = "increaseBirthdate"; public static final String PROCESS_NAME_ADD_TO_PEOPLES_AGE = "addToPeoplesAge"; public static final String TABLE_NAME_PERSON_FILE = "personFile"; public static final String TABLE_NAME_PERSON_MEMORY = "personMemory"; @@ -144,6 +149,7 @@ public class TestUtils qInstance.addProcess(defineProcessAddToPeoplesAge()); qInstance.addProcess(new BasicETLProcess().defineProcessMetaData()); qInstance.addProcess(new StreamedETLProcess().defineProcessMetaData()); + qInstance.addProcess(defineProcessIncreasePersonBirthdate()); qInstance.addAutomationProvider(definePollingAutomationProvider()); @@ -154,6 +160,72 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + private static QProcessMetaData defineProcessIncreasePersonBirthdate() + { + return new QProcessMetaData() + .withName(PROCESS_NAME_INCREASE_BIRTHDATE) + .withTableName(TABLE_NAME_PERSON_MEMORY) + + .addStep(new QFrontendStepMetaData() + .withName("preview") + ) + + .addStep(new QBackendStepMetaData() + .withName("doWork") + .withCode(new QCodeReference(IncreaseBirthdateStep.class)) + .withInputData(new QFunctionInputMetaData() + .withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON_MEMORY))) + .withOutputMetaData(new QFunctionOutputMetaData() + .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING).withDefaultValue("Success!")))) + ) + + .addStep(new QFrontendStepMetaData() + .withName("results") + .withFormField(new QFieldMetaData("outputMessage", QFieldType.STRING)) + ); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class IncreaseBirthdateStep implements BackendStep + { + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + List recordsToUpdate = new ArrayList<>(); + for(QRecord record : runBackendStepInput.getRecords()) + { + LocalDate birthDate = record.getValueLocalDate("birthDate"); + + if(birthDate != null && birthDate.getYear() < 1900) + { + recordsToUpdate.add(new QRecord() + .withValue("id", record.getValue("id")) + .withValue("birthDate", birthDate.withYear(1900)) + ); + } + } + + UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance()); + updateInput.setSession(runBackendStepInput.getSession()); + updateInput.setTableName(TABLE_NAME_PERSON_MEMORY); + updateInput.setRecords(recordsToUpdate); + new UpdateAction().execute(updateInput); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -375,6 +447,16 @@ public class TestUtils *******************************************************************************/ public static QTableMetaData definePersonMemoryTable() { + ///////////////////////////////////////////////////////////////////////////// + // the checkAge automation will only run on persons younger than this date // + ///////////////////////////////////////////////////////////////////////////// + LocalDate youngPersonLimitDate = LocalDate.now().minusYears(18); + + ///////////////////////////////////////////////////////////////////////////////////// + // the increaseBirthdate automation will only run on persons born before this date // + ///////////////////////////////////////////////////////////////////////////////////// + LocalDate increaseBirthdateLimitDate = LocalDate.of(1900, Month.JANUARY, 1); + return (new QTableMetaData() .withName(TABLE_NAME_PERSON_MEMORY) .withBackendName(MEMORY_BACKEND_NAME) @@ -386,12 +468,19 @@ public class TestUtils .withAction(new TableAutomationAction() .withName("checkAgeOnInsert") .withTriggerEvent(TriggerEvent.POST_INSERT) + .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, List.of(youngPersonLimitDate)))) .withCodeReference(new QCodeReference(CheckAge.class)) ) + .withAction(new TableAutomationAction() + .withName("increaseBirthdate") + .withTriggerEvent(TriggerEvent.POST_INSERT) + .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, List.of(increaseBirthdateLimitDate)))) + .withProcessName(PROCESS_NAME_INCREASE_BIRTHDATE) + ) .withAction(new TableAutomationAction() .withName("logOnUpdatePerFilter") .withTriggerEvent(TriggerEvent.POST_UPDATE) - .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Darin")))) + .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.CONTAINS, List.of("Darin")))) .withCodeReference(new QCodeReference(LogPersonUpdate.class)) ) ); @@ -420,7 +509,6 @@ public class TestUtils LocalDate birthDate = record.getValueLocalDate("birthDate"); if(birthDate != null && birthDate.isAfter(limitDate)) { - LOG.info("Person [" + record.getValueInteger("id") + "] is a minor - updating their firstName to state such."); recordsToUpdate.add(new QRecord() .withValue("id", record.getValue("id")) .withValue("firstName", record.getValueString("firstName") + SUFFIX_FOR_MINORS) From 3a69ce7d2f25062f24970ea53ff3b02c197c9578 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 1 Sep 2022 15:56:37 -0500 Subject: [PATCH 21/28] QQQ-40 getting closer to production-ready on automations --- .../core/actions/async/AsyncJobManager.java | 4 +- .../actions/async/AsyncRecordPipeLoop.java | 16 +- .../RecordAutomationStatusUpdater.java | 54 +++- .../polling/PollingAutomationRunner.java | 225 +++++++++++----- .../core/actions/reporting/RecordPipe.java | 2 +- .../qqq/backend/core/model/data/QRecord.java | 11 + .../metadata/processes/QProcessMetaData.java | 30 ++- .../memory/MemoryBackendModule.java | 4 + .../memory/MemoryRecordStore.java | 251 +++++++++++++++--- .../StreamedETLPreviewStep.java | 28 ++ .../src/main/resources/log4j2.xml | 4 +- .../PollingAutomationExecutorTest.java | 60 +---- .../polling/PollingAutomationRunnerTest.java | 226 ++++++++++++++++ .../actions/reporting/ReportActionTest.java | 2 +- .../memory/MemoryBackendModuleTest.java | 98 +++++++ .../qqq/backend/core/utils/TestUtils.java | 94 ++++++- 16 files changed, 933 insertions(+), 176 deletions(-) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunnerTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java index fbf95f07..041629e3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java @@ -135,11 +135,11 @@ public class AsyncJobManager Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8)); try { - LOG.info("Starting job " + uuidAndTypeStateKey.getUuid()); + LOG.debug("Starting job " + uuidAndTypeStateKey.getUuid()); T result = asyncJob.run(new AsyncJobCallback(uuidAndTypeStateKey.getUuid(), asyncJobStatus)); asyncJobStatus.setState(AsyncJobState.COMPLETE); getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus); - LOG.info("Completed job " + uuidAndTypeStateKey.getUuid()); + LOG.debug("Completed job " + uuidAndTypeStateKey.getUuid()); return (result); } catch(Exception e) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java index a15948f3..d41782bd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java @@ -48,7 +48,7 @@ public class AsyncRecordPipeLoop /////////////////////////////////////////////////// AsyncJobManager asyncJobManager = new AsyncJobManager(); String jobUUID = asyncJobManager.startJob(jobName, supplier::apply); - LOG.info("Started supplier job [" + jobUUID + "] for record pipe."); + LOG.debug("Started supplier job [" + jobUUID + "] for record pipe."); AsyncJobState jobState = AsyncJobState.RUNNING; AsyncJobStatus asyncJobStatus = null; @@ -66,7 +66,7 @@ public class AsyncRecordPipeLoop // if the pipe is too empty, sleep to let the producer work. // // todo - smarter sleep? like get notified vs. sleep? // /////////////////////////////////////////////////////////////// - LOG.debug("Too few records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work"); + LOG.trace("Too few records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work"); SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS); nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS); @@ -85,7 +85,7 @@ public class AsyncRecordPipeLoop nextSleepMillis = INIT_SLEEP_MS; recordCount += consumer.get(); - LOG.info(String.format("Processed %,d records so far", recordCount)); + LOG.debug(String.format("Processed %,d records so far", recordCount)); if(recordLimit != null && recordCount >= recordLimit) { @@ -117,7 +117,7 @@ public class AsyncRecordPipeLoop jobState = asyncJobStatus.getState(); } - LOG.info("Job [" + jobUUID + "] completed with status: " + asyncJobStatus); + LOG.debug("Job [" + jobUUID + "][" + jobName + "] completed with status: " + asyncJobStatus); /////////////////////////////////// // propagate errors from the job // @@ -133,8 +133,12 @@ public class AsyncRecordPipeLoop recordCount += consumer.get(); long endTime = System.currentTimeMillis(); - LOG.info(String.format("Processed %,d records", recordCount) - + String.format(" at end of job in %,d ms (%.2f records/second).", (endTime - jobStartTime), 1000d * (recordCount / (.001d + (endTime - jobStartTime))))); + + if(recordCount > 0) + { + LOG.info(String.format("Processed %,d records", recordCount) + + String.format(" at end of job [%s] in %,d ms (%.2f records/second).", jobName, (endTime - jobStartTime), 1000d * (recordCount / (.001d + (endTime - jobStartTime))))); + } return (recordCount); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java index 8728f4a6..d31b7297 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java @@ -1,7 +1,9 @@ package com.kingsrook.qqq.backend.core.actions.automation; +import java.util.ArrayList; import java.util.List; +import java.util.Objects; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; @@ -10,6 +12,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import org.apache.commons.lang.NotImplementedException; @@ -37,7 +41,12 @@ public class RecordAutomationStatusUpdater return (false); } - if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS) || automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS)) + if(canWeSkipPendingAndGoToOkay(table, automationStatus)) + { + automationStatus = AutomationStatus.OK; + } + + if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS)) { Exception e = new Exception(); for(StackTraceElement stackTraceElement : e.getStackTrace()) @@ -45,7 +54,7 @@ public class RecordAutomationStatusUpdater String className = stackTraceElement.getClassName(); if(className.contains("com.kingsrook.qqq.backend.core.actions.automation") && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test")) { - LOG.info("Avoiding re-setting automation status to PENDING while running an automation"); + LOG.debug("Avoiding re-setting automation status to PENDING_UPDATE while running an automation"); return (false); } } @@ -66,6 +75,35 @@ public class RecordAutomationStatusUpdater + /******************************************************************************* + ** If a table has no automation actions defined for Insert (or Update), and we're + ** being asked to set status to PENDING_INSERT (or PENDING_UPDATE), then just + ** move the status straight to OK. + *******************************************************************************/ + private static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus) + { + List tableActions = Objects.requireNonNullElse(table.getAutomationDetails().getActions(), new ArrayList<>()); + + if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS)) + { + if(tableActions.stream().noneMatch(a -> TriggerEvent.POST_INSERT.equals(a.getTriggerEvent()))) + { + return (true); + } + } + else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS)) + { + if(tableActions.stream().noneMatch(a -> TriggerEvent.POST_UPDATE.equals(a.getTriggerEvent()))) + { + return (true); + } + } + + return (false); + } + + + /******************************************************************************* ** for a list of records, update their automation status and actually Update the ** backend as well. @@ -81,7 +119,17 @@ public class RecordAutomationStatusUpdater UpdateInput updateInput = new UpdateInput(instance); updateInput.setSession(session); updateInput.setTableName(table.getName()); - updateInput.setRecords(records); + + ///////////////////////////////////////////////////////////////////////////////////// + // build records with just their pkey & status field for this update, to avoid // + // changing other values (relies on assumption of Patch semantics in UpdateAction) // + ///////////////////////////////////////////////////////////////////////////////////// + updateInput.setRecords(records.stream().map(r -> new QRecord() + .withTableName(r.getTableName()) + .withValue(table.getPrimaryKeyField(), r.getValue(table.getPrimaryKeyField())) + .withValue(automationDetails.getStatusTracking().getFieldName(), r.getValue(automationDetails.getStatusTracking().getFieldName()))).toList()); + updateInput.setAreAllValuesBeingUpdatedTheSame(true); + new UpdateAction().execute(updateInput); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java index 46d69208..06f31946 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java @@ -1,33 +1,41 @@ package com.kingsrook.qqq.backend.core.actions.automation.polling; +import java.io.Serializable; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Supplier; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent; import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; -import org.apache.commons.lang.NotImplementedException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -40,8 +48,8 @@ class PollingAutomationRunner implements Runnable { private static final Logger LOG = LogManager.getLogger(PollingAutomationRunner.class); - private QInstance instance; - private String providerName; + private QInstance instance; + private String providerName; private Supplier sessionSupplier; private List managedTables = new ArrayList<>(); @@ -130,7 +138,7 @@ class PollingAutomationRunner implements Runnable /******************************************************************************* - ** + ** Query for and process records that have a PENDING status on a given table. *******************************************************************************/ private void processTable(QTableMetaData table) throws QException { @@ -142,7 +150,7 @@ class PollingAutomationRunner implements Runnable /******************************************************************************* - ** + ** Query for and process records that have a PENDING_INSERT or PENDING_UPDATE status on a given table. *******************************************************************************/ private void processTableInsertOrUpdate(QTableMetaData table, QSession session, boolean isInsert) throws QException { @@ -153,88 +161,185 @@ class PollingAutomationRunner implements Runnable return; } - LOG.info(" Query for records " + automationStatus + " in " + table); + LOG.debug(" Query for records " + automationStatus + " in " + table); - QueryInput queryInput = new QueryInput(instance); - queryInput.setSession(session); // todo - where the heck can we get this from?? - queryInput.setTableName(table.getName()); + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // run an async-pipe loop - that will query for records in PENDING - put them in a pipe - then apply actions to them // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + RecordPipe recordPipe = new RecordPipe(); + AsyncRecordPipeLoop asyncRecordPipeLoop = new AsyncRecordPipeLoop(); + asyncRecordPipeLoop.run("PollingAutomationRunner>Query>" + (isInsert ? "insert" : "update"), null, recordPipe, (status) -> + { + QueryInput queryInput = new QueryInput(instance); + queryInput.setSession(session); + queryInput.setTableName(table.getName()); + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(table.getAutomationDetails().getStatusTracking().getFieldName(), QCriteriaOperator.IN, List.of(automationStatus.getId())))); + queryInput.setRecordPipe(recordPipe); + return (new QueryAction().execute(queryInput)); + }, () -> + { + List records = recordPipe.consumeAvailableRecords(); + applyActionsToRecords(session, table, records, actions, isInsert); + return (records.size()); + } + ); + } + + + /******************************************************************************* + ** For a set of records that were found to be in a PENDING state - run all the + ** table's actions against them. + *******************************************************************************/ + private void applyActionsToRecords(QSession session, QTableMetaData table, List records, List actions, boolean isInsert) throws QException + { + if(CollectionUtils.nullSafeIsEmpty(records)) + { + return; + } + + /////////////////////////////////////////////////// + // mark the records as RUNNING their automations // + /////////////////////////////////////////////////// + RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, isInsert ? AutomationStatus.RUNNING_INSERT_AUTOMATIONS : AutomationStatus.RUNNING_UPDATE_AUTOMATIONS); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // foreach action - run it against the records (but only if they match the action's filter, if there is one) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + boolean anyActionsFailed = false; for(TableAutomationAction action : actions) { - QQueryFilter filter = action.getFilter(); - if(filter == null) + try { - filter = new QQueryFilter(); + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // note - this method - will re-query the objects, so we should have confidence that their data is fresh... // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List matchingQRecords = getRecordsMatchingActionFilter(session, table, records, action); + LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action); + if(CollectionUtils.nullSafeHasContents(matchingQRecords)) + { + LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action); + applyActionToMatchingRecords(session, table, matchingQRecords, action); + } } - - filter.addCriteria(new QFilterCriteria(table.getAutomationDetails().getStatusTracking().getFieldName(), QCriteriaOperator.IN, List.of(automationStatus.getId()))); - queryInput.setFilter(filter); - - QueryOutput queryOutput = new QueryAction().execute(queryInput); - - // todo - pipe this query!! - - if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords())) + catch(Exception e) { - LOG.info(" Processing " + queryOutput.getRecords().size() + " records in " + table + " for action " + action); - processRecords(table, actions, queryOutput.getRecords(), session, isInsert); + LOG.warn("Caught exception processing records on " + table + " for action " + action, e); + anyActionsFailed = true; } } + + //////////////////////////////////////// + // update status on all these records // + //////////////////////////////////////// + if(anyActionsFailed) + { + RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.FAILED_UPDATE_AUTOMATIONS); + } + else + { + RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.OK); + } } /******************************************************************************* + ** For a given action, and a list of records - return a new list, of the ones + ** which match the action's filter (if there is one - if not, then all match). ** + ** Note that this WILL re-query the objects (ALWAYS - even if the action has no filter). + ** This has the nice side effect of always giving fresh/updated records, despite having + ** some cost. + ** + ** At one point, we considered just applying the filter using java-comparisons, + ** but that will almost certainly give potentially different results than a true + ** backend - e.g., just consider if the DB is case-sensitive for strings... *******************************************************************************/ - private void processRecords(QTableMetaData table, List actions, List records, QSession session, boolean isInsert) throws QException + private List getRecordsMatchingActionFilter(QSession session, QTableMetaData table, List records, TableAutomationAction action) throws QException { - try - { - updateRecordAutomationStatus(table, session, records, isInsert ? AutomationStatus.RUNNING_INSERT_AUTOMATIONS : AutomationStatus.RUNNING_UPDATE_AUTOMATIONS); + QueryInput queryInput = new QueryInput(instance); + queryInput.setSession(session); + queryInput.setTableName(table.getName()); - for(TableAutomationAction action : actions) + QQueryFilter filter = new QQueryFilter(); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // copy filter criteria from the action's filter to a new filter that we'll run here. // + // Critically - don't modify the filter object on the action! as that object has a long lifespan. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + if(action.getFilter() != null) + { + if(action.getFilter().getCriteria() != null) { - //////////////////////////////////// - // todo - what, re-query them? :( // - //////////////////////////////////// - if(StringUtils.hasContent(action.getProcessName())) - { - ////////////////////////////////////////////////////////////////////////////////////////////// - // todo - uh, how to make these records the input, where an extract step might be involved? // - // should extract step ... see record list and just use it? i think maybe? // - ////////////////////////////////////////////////////////////////////////////////////////////// - throw (new NotImplementedException("processes for automation not yet implemented")); - } - else if(action.getCodeReference() != null) - { - LOG.info(" Executing action: [" + action.getName() + "] as code reference: " + action.getCodeReference()); - RecordAutomationInput input = new RecordAutomationInput(instance); - input.setSession(session); - input.setTableName(table.getName()); - input.setRecordList(records); - - RecordAutomationHandler recordAutomationHandler = QCodeLoader.getRecordAutomationHandler(action); - recordAutomationHandler.execute(input); - } + action.getFilter().getCriteria().forEach(filter::addCriteria); } + if(action.getFilter().getOrderBys() != null) + { + action.getFilter().getOrderBys().forEach(filter::addOrderBy); + } + } - updateRecordAutomationStatus(table, session, records, AutomationStatus.OK); - } - catch(Exception e) - { - updateRecordAutomationStatus(table, session, records, isInsert ? AutomationStatus.FAILED_INSERT_AUTOMATIONS : AutomationStatus.FAILED_UPDATE_AUTOMATIONS); - } + filter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList())); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // always add order-by the primary key, to give more predictable/consistent results // + // todo - in future - if this becomes a source of slowness, make this a config to opt-out? // + ///////////////////////////////////////////////////////////////////////////////////////////// + filter.addOrderBy(new QFilterOrderBy().withFieldName(table.getPrimaryKeyField())); + + queryInput.setFilter(filter); + + return (new QueryAction().execute(queryInput).getRecords()); } /******************************************************************************* - ** + ** Finally, actually run action code against a list of known matching records. *******************************************************************************/ - private void updateRecordAutomationStatus(QTableMetaData table, QSession session, List records, AutomationStatus automationStatus) throws QException + private void applyActionToMatchingRecords(QSession session, QTableMetaData table, List records, TableAutomationAction action) throws Exception { - RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, automationStatus); + if(StringUtils.hasContent(action.getProcessName())) + { + RunProcessInput runProcessInput = new RunProcessInput(instance); + runProcessInput.setSession(session); + runProcessInput.setProcessName(action.getProcessName()); + runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // kinda hacky - if we see that this process has an input field of a given name, then put a filter in there to find these records... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QProcessMetaData process = instance.getProcess(action.getProcessName()); + if(process.getInputFields().stream().anyMatch(f -> f.getName().equals(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER))) + { + List recordIds = records.stream().map(r -> r.getValueInteger(table.getPrimaryKeyField())).collect(Collectors.toList()); + QQueryFilter queryFilter = new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds)); + runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, queryFilter); + } + else + { + runProcessInput.setRecords(records); + } + + RunProcessAction runProcessAction = new RunProcessAction(); + RunProcessOutput runProcessOutput = runProcessAction.execute(runProcessInput); + if(runProcessOutput.getException().isPresent()) + { + throw (runProcessOutput.getException().get()); + } + } + else if(action.getCodeReference() != null) + { + LOG.debug(" Executing action: [" + action.getName() + "] as code reference: " + action.getCodeReference()); + RecordAutomationInput input = new RecordAutomationInput(instance); + input.setSession(session); + input.setTableName(table.getName()); + input.setRecordList(records); + + RecordAutomationHandler recordAutomationHandler = QCodeLoader.getRecordAutomationHandler(action); + recordAutomationHandler.execute(input); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java index 87b348b7..548e8869 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java @@ -118,7 +118,7 @@ public class RecordPipe LOG.warn("Giving up adding record to pipe, due to pipe being full for more than {} millis", MAX_SLEEP_LOOP_MILLIS); throw (new IllegalStateException("Giving up adding record to pipe, due to pipe staying full too long.")); } - LOG.debug("Record pipe.add failed (due to full pipe). Blocking."); + LOG.trace("Record pipe.add failed (due to full pipe). Blocking."); SleepUtils.sleep(BLOCKING_SLEEP_MILLIS, TimeUnit.MILLISECONDS); offerResult = queue.offer(record); now = System.currentTimeMillis(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index 29903fc3..4206c97b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -104,6 +104,17 @@ public class QRecord implements Serializable + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return "QRecord{tableName='" + tableName + "',id='" + getValue("id") + "'}"; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index 20b40e02..45247881 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -24,8 +24,10 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; @@ -273,17 +275,25 @@ public class QProcessMetaData implements QAppChildMetaData /******************************************************************************* - ** Get a list of all of the input fields used by all the steps in this process. + ** Get a list of all the *unique* input fields used by all the steps in this process. *******************************************************************************/ @JsonIgnore public List getInputFields() { - List rs = new ArrayList<>(); + Set usedFieldNames = new HashSet<>(); + List rs = new ArrayList<>(); if(steps != null) { for(QStepMetaData step : steps.values()) { - rs.addAll(step.getInputFields()); + for(QFieldMetaData field : step.getInputFields()) + { + if(!usedFieldNames.contains(field.getName())) + { + rs.add(field); + usedFieldNames.add(field.getName()); + } + } } } return (rs); @@ -292,17 +302,25 @@ public class QProcessMetaData implements QAppChildMetaData /******************************************************************************* - ** Get a list of all of the output fields used by all the steps in this process. + ** Get a list of all the *unique* output fields used by all the steps in this process. *******************************************************************************/ @JsonIgnore public List getOutputFields() { - List rs = new ArrayList<>(); + Set usedFieldNames = new HashSet<>(); + List rs = new ArrayList<>(); if(steps != null) { for(QStepMetaData step : steps.values()) { - rs.addAll(step.getOutputFields()); + for(QFieldMetaData field : step.getOutputFields()) + { + if(!usedFieldNames.contains(field.getName())) + { + rs.add(field); + usedFieldNames.add(field.getName()); + } + } } } return (rs); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java index 84852cb9..30346a7f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java @@ -35,6 +35,10 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; ** A simple (probably only valid for testing?) implementation of the QModuleInterface, ** that just stores its records in-memory. ** + ** In general, this class is intended to behave, as much as possible, like an RDBMS. + ** + ** TODO - in future, if we need to - make configs for things like "case-insensitive", + ** and "allow loose typing". *******************************************************************************/ public class MemoryBackendModule implements QBackendModuleInterface { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java index 90eedd28..8dbd9d07 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory; import java.io.Serializable; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -32,12 +33,14 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import org.apache.commons.lang.NotImplementedException; @@ -121,42 +124,7 @@ public class MemoryRecordStore for(QRecord qRecord : tableData.values()) { - boolean recordMatches = true; - if(input.getFilter() != null && input.getFilter().getCriteria() != null) - { - for(QFilterCriteria criterion : input.getFilter().getCriteria()) - { - String fieldName = criterion.getFieldName(); - Serializable value = qRecord.getValue(fieldName); - switch(criterion.getOperator()) - { - case EQUALS: - { - if(!value.equals(criterion.getValues().get(0))) - { - recordMatches = false; - } - break; - } - case IN: - { - if(!criterion.getValues().contains(value)) - { - recordMatches = false; - } - break; - } - default: - { - throw new NotImplementedException("Operator [" + criterion.getOperator() + "] is not yet implemented in the Memory backend."); - } - } - if(!recordMatches) - { - break; - } - } - } + boolean recordMatches = doesRecordMatch(input.getFilter(), qRecord); if(recordMatches) { @@ -169,6 +137,217 @@ public class MemoryRecordStore + /******************************************************************************* + ** + *******************************************************************************/ + private boolean doesRecordMatch(QQueryFilter filter, QRecord qRecord) + { + boolean recordMatches = true; + if(filter != null && filter.getCriteria() != null) + { + for(QFilterCriteria criterion : filter.getCriteria()) + { + String fieldName = criterion.getFieldName(); + Serializable value = qRecord.getValue(fieldName); + + switch(criterion.getOperator()) + { + case EQUALS: + { + recordMatches = testEquals(criterion, value); + break; + } + case NOT_EQUALS: + { + recordMatches = !testEquals(criterion, value); + break; + } + case IN: + { + recordMatches = testIn(criterion, value); + break; + } + case NOT_IN: + { + recordMatches = !testIn(criterion, value); + break; + } + case CONTAINS: + { + recordMatches = testContains(criterion, fieldName, value); + break; + } + case NOT_CONTAINS: + { + recordMatches = !testContains(criterion, fieldName, value); + break; + } + case GREATER_THAN: + { + recordMatches = testGreaterThan(criterion, value); + break; + } + case GREATER_THAN_OR_EQUALS: + { + recordMatches = testGreaterThan(criterion, value) || testEquals(criterion, value); + break; + } + case LESS_THAN: + { + recordMatches = !testGreaterThan(criterion, value) && !testEquals(criterion, value); + break; + } + case LESS_THAN_OR_EQUALS: + { + recordMatches = !testGreaterThan(criterion, value); + break; + } + default: + { + throw new NotImplementedException("Operator [" + criterion.getOperator() + "] is not yet implemented in the Memory backend."); + } + } + if(!recordMatches) + { + break; + } + } + } + return recordMatches; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean testGreaterThan(QFilterCriteria criterion, Serializable value) + { + Serializable criterionValue = criterion.getValues().get(0); + if(criterionValue == null) + { + throw (new IllegalArgumentException("Missing criterion value in query")); + } + + if (value == null) + { + ///////////////////////////////////////////////////////////////////////////////////// + // a database would say 'false' for if a null column is > a value, so do the same. // + ///////////////////////////////////////////////////////////////////////////////////// + return (false); + } + + if(value instanceof LocalDate valueDate && criterionValue instanceof LocalDate criterionValueDate) + { + return (valueDate.isAfter(criterionValueDate)); + } + + if(value instanceof Number valueNumber && criterionValue instanceof Number criterionValueNumber) + { + return (valueNumber.doubleValue() > criterionValueNumber.doubleValue()); + } + + throw (new NotImplementedException("Greater/Less Than comparisons are not (yet?) implemented for the supplied types [" + value.getClass().getSimpleName() + "][" + criterionValue.getClass().getSimpleName() + "]")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean testIn(QFilterCriteria criterion, Serializable value) + { + if(!criterion.getValues().contains(value)) + { + return (false); + } + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean testEquals(QFilterCriteria criterion, Serializable value) + { + if(value == null) + { + return (false); + } + + if(!value.equals(criterion.getValues().get(0))) + { + return (false); + } + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean testContains(QFilterCriteria criterion, String fieldName, Serializable value) + { + String stringValue = getStringFieldValue(value, fieldName, criterion); + String criterionValue = getFirstStringCriterionValue(criterion); + + if(!stringValue.contains(criterionValue)) + { + return (false); + } + + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getFirstStringCriterionValue(QFilterCriteria criteria) + { + if(CollectionUtils.nullSafeIsEmpty(criteria.getValues())) + { + throw new IllegalArgumentException("Missing value for [" + criteria.getOperator() + "] criteria on field [" + criteria.getFieldName() + "]"); + } + Serializable value = criteria.getValues().get(0); + if(value == null) + { + return ""; + } + + if(!(value instanceof String stringValue)) + { + throw new ClassCastException("Value [" + value + "] for criteria [" + criteria.getFieldName() + "] is not a String, which is required for the [" + criteria.getOperator() + "] operator."); + } + + return (stringValue); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getStringFieldValue(Serializable value, String fieldName, QFilterCriteria criterion) + { + if(value == null) + { + return ""; + } + + if(!(value instanceof String stringValue)) + { + throw new ClassCastException("Value [" + value + "] in field [" + fieldName + "] is not a String, which is required for the [" + criterion.getOperator() + "] operator."); + } + + return (stringValue); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java index d16c40b5..59eb69c1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -65,6 +65,15 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe return; } + ///////////////////////////////////////////////////////////////// + // if we're running inside an automation, then skip this step. // + ///////////////////////////////////////////////////////////////// + if(runningWithinAutomation()) + { + LOG.info("Skipping preview step when [" + runBackendStepInput.getProcessName() + "] is running as part of an automation."); + return; + } + /////////////////////////////////////////// // request a count from the extract step // /////////////////////////////////////////// @@ -109,6 +118,25 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe + /******************************************************************************* + ** + *******************************************************************************/ + private boolean runningWithinAutomation() + { + Exception e = new Exception(); + for(StackTraceElement stackTraceElement : e.getStackTrace()) + { + String className = stackTraceElement.getClassName(); + if(className.contains("com.kingsrook.qqq.backend.core.actions.automation")) + { + return (true); + } + } + return false; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/resources/log4j2.xml b/qqq-backend-core/src/main/resources/log4j2.xml index d69b17bc..a16e77af 100644 --- a/qqq-backend-core/src/main/resources/log4j2.xml +++ b/qqq-backend-core/src/main/resources/log4j2.xml @@ -2,11 +2,11 @@ - + - + diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java index cfda8981..a9cbb572 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java @@ -11,12 +11,10 @@ import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; -import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -99,56 +97,6 @@ class PollingAutomationExecutorTest - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testUpdate() throws QException - { - QInstance qInstance = TestUtils.defineInstance(); - - /////////////////////////////////////////////////////////////////////////////// - // insert 2 people - one who should be logged by logger-on-update automation // - /////////////////////////////////////////////////////////////////////////////// - InsertInput insertInput = new InsertInput(qInstance); - insertInput.setSession(new QSession()); - insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); - insertInput.setRecords(List.of( - new QRecord().withValue("id", 1).withValue("firstName", "Tim"), - new QRecord().withValue("id", 2).withValue("firstName", "Darin") - )); - new InsertAction().execute(insertInput); - - //////////////////////////////////////////////// - // have the polling executor run "for awhile" // - //////////////////////////////////////////////// - runPollingAutomationExecutorForAwhile(qInstance); - - ////////////////////////////////////////////////// - // assert that the update-automation didn't run // - ////////////////////////////////////////////////// - assertThat(TestUtils.LogPersonUpdate.updatedIds).isNullOrEmpty(); - - UpdateInput updateInput = new UpdateInput(qInstance); - updateInput.setSession(new QSession()); - updateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); - updateInput.setRecords(List.of( - new QRecord().withValue("id", 1).withValue("lastName", "now with a LastName"), - new QRecord().withValue("id", 2).withValue("lastName", "now with a LastName") - )); - new UpdateAction().execute(updateInput); - - //////////////////////////////////////////////// - // have the polling executor run "for awhile" // - //////////////////////////////////////////////// - runPollingAutomationExecutorForAwhile(qInstance); - - /////////////////////////////////////////////////// - // assert that the update-automation DID run now // - /////////////////////////////////////////////////// - assertThat(TestUtils.LogPersonUpdate.updatedIds).contains(2); - } - /******************************************************************************* @@ -166,14 +114,14 @@ class PollingAutomationExecutorTest .getAutomationDetails().getActions().get(0) .setCodeReference(new QCodeReference(CaptureSessionIdAutomationHandler.class)); - ///////////////////// - // insert a person // - ///////////////////// + //////////////////////////////////////////////////////////// + // insert a person that will trigger the on-insert action // + //////////////////////////////////////////////////////////// InsertInput insertInput = new InsertInput(qInstance); insertInput.setSession(new QSession()); insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); insertInput.setRecords(List.of( - new QRecord().withValue("id", 1).withValue("firstName", "Tim") + new QRecord().withValue("id", 1).withValue("firstName", "Tim").withValue("birthDate", LocalDate.now()) )); new InsertAction().execute(insertInput); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunnerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunnerTest.java new file mode 100644 index 00000000..0b5da000 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunnerTest.java @@ -0,0 +1,226 @@ +package com.kingsrook.qqq.backend.core.actions.automation.polling; + + +import java.time.LocalDate; +import java.time.Month; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for PollingAutomationRunner + *******************************************************************************/ +class PollingAutomationRunnerTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** Test a cycle that does an insert, some automations, then and an update, and more automations. + *******************************************************************************/ + @Test + void testInsertAndUpdate() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, TestUtils.POLLING_AUTOMATION, null); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // insert 2 person records, one who should be both updated by the insert action, and should be logged by logger-on-update automation // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Tim").withValue("birthDate", LocalDate.now()), + new QRecord().withValue("id", 2).withValue("firstName", "Darin") + )); + new InsertAction().execute(insertInput); + assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS); + + ////////////////////////////////////////////////////////////////////////////////////////// + // assert that the update-automation won't run - as no UPDATE has happened on the table // + // even though the insert action does update the records!! // + ////////////////////////////////////////////////////////////////////////////////////////// + pollingAutomationRunner.run(); + assertThat(TestUtils.LogPersonUpdate.updatedIds).isNullOrEmpty(); + assertAllRecordsAutomationStatus(AutomationStatus.OK); + + //////////////////////////////////////////// + // make sure the minor person was updated // + //////////////////////////////////////////// + Optional updatedMinorRecord = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY).stream().filter(r -> r.getValueInteger("id").equals(1)).findFirst(); + assertThat(updatedMinorRecord) + .isPresent() + .get() + .extracting(r -> r.getValueString("firstName")) + .isEqualTo("Tim" + TestUtils.CheckAge.SUFFIX_FOR_MINORS); + + ///////////////////////////////////////////////////////////////////////////////////////// + // run automations again - make sure that there haven't been any updates triggered yet // + ///////////////////////////////////////////////////////////////////////////////////////// + pollingAutomationRunner.run(); + assertThat(TestUtils.LogPersonUpdate.updatedIds).isNullOrEmpty(); + assertAllRecordsAutomationStatus(AutomationStatus.OK); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now do an user-driven update - this SHOULD trigger the update automation next time we run automations. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + UpdateInput updateInput = new UpdateInput(qInstance); + updateInput.setSession(new QSession()); + updateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + updateInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("lastName", "now with a LastName"), + new QRecord().withValue("id", 2).withValue("lastName", "now with a LastName") + )); + new UpdateAction().execute(updateInput); + assertAllRecordsAutomationStatus(AutomationStatus.PENDING_UPDATE_AUTOMATIONS); + + ///////////////////////////////////////////////////////////////////////////////// + // assert that the update-automation DOES run now - and that it only runs once // + // note that it will only run on a sub-set of the records // + ///////////////////////////////////////////////////////////////////////////////// + pollingAutomationRunner.run(); + assertThat(TestUtils.LogPersonUpdate.updatedIds) + .contains(2) + .hasSize(1); + assertAllRecordsAutomationStatus(AutomationStatus.OK); + + ///////////////////////////////////////////////////// + // re-run and assert no further automations happen // + ///////////////////////////////////////////////////// + TestUtils.LogPersonUpdate.updatedIds.clear(); + pollingAutomationRunner.run(); + assertThat(TestUtils.LogPersonUpdate.updatedIds).isNullOrEmpty(); + assertAllRecordsAutomationStatus(AutomationStatus.OK); + } + + + + /******************************************************************************* + ** Test a large-ish number - to demonstrate paging working. + ** + ** Note - this caught an issue during original development, where the QueryFilter + ** attached to the Action was being re-used, w/ new "id IN *" criteria being re-added + ** to it - so, good test. + *******************************************************************************/ + @Test + void testMultiPages() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, TestUtils.POLLING_AUTOMATION, null); + + ////////////////////////////////////////////////////////////////////////////////// + // insert many people - half who should be updated by the AgeChecker automation // + ////////////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + + insertInput.setRecords(new ArrayList<>()); + int SIZE = 2_500; + for(int i = 0; i < SIZE; i++) + { + insertInput.getRecords().add(new QRecord().withValue("firstName", "Tim").withValue("lastName", "Number " + i).withValue("birthDate", LocalDate.now())); + insertInput.getRecords().add(new QRecord().withValue("firstName", "Darin").withValue("lastName", "Number " + i)); + } + + new InsertAction().execute(insertInput); + assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS); + + ///////////////////////// + // run the automations // + ///////////////////////// + pollingAutomationRunner.run(); + assertAllRecordsAutomationStatus(AutomationStatus.OK); + + /////////////////////////////////////////////////////////////////////////// + // make sure that all 'minor' persons were updated (e.g., all the Tim's) // + /////////////////////////////////////////////////////////////////////////// + int updatedMinorsCount = 0; + for(QRecord qRecord : TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)) + { + if(qRecord.getValueString("firstName").startsWith("Tim")) + { + assertEquals("Tim" + TestUtils.CheckAge.SUFFIX_FOR_MINORS, qRecord.getValueString("firstName")); + updatedMinorsCount++; + } + } + + assertEquals(SIZE, updatedMinorsCount, "Expected number of updated records"); + } + + + + /******************************************************************************* + ** Test a cycle that does an insert, some automations, then and an update, and more automations. + *******************************************************************************/ + @Test + void testRunningProcess() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, TestUtils.POLLING_AUTOMATION, null); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // insert 2 person records, one who should be both updated by the insert action, and should be logged by logger-on-update automation // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Tim").withValue("birthDate", LocalDate.of(1886, Month.JUNE, 6)), + new QRecord().withValue("id", 2).withValue("firstName", "Darin").withValue("birthDate", LocalDate.of(1904, Month.APRIL, 4)) + )); + new InsertAction().execute(insertInput); + + pollingAutomationRunner.run(); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // make sure the process ran - which means, it would have updated Tim's birth year to 1900 // + ///////////////////////////////////////////////////////////////////////////////////////////// + Optional updatedMinorRecord = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY).stream().filter(r -> r.getValueInteger("id").equals(1)).findFirst(); + assertThat(updatedMinorRecord) + .isPresent() + .get() + .extracting(r -> r.getValueLocalDate("birthDate").getYear()) + .isEqualTo(1900); + + } + + + /******************************************************************************* + ** + *******************************************************************************/ + private void assertAllRecordsAutomationStatus(AutomationStatus pendingInsertAutomations) throws QException + { + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)) + .isNotEmpty() + .allMatch(r -> pendingInsertAutomations.getId().equals(r.getValue(TestUtils.standardQqqAutomationStatusField().getName()))); + } +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java index 08d0d5ba..aabef749 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java @@ -81,7 +81,7 @@ class ReportActionTest public void testBigger() throws Exception { // int recordCount = 2_000_000; // to really stress locally, use this. - int recordCount = 50_000; + int recordCount = 10_000; String filename = "/tmp/ReportActionTest.csv"; runReport(recordCount, filename, ReportFormat.CSV, false); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java index 9bc5081f..567129b9 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModuleTest.java @@ -22,6 +22,8 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory; +import java.time.LocalDate; +import java.time.Month; import java.util.List; import java.util.function.Function; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; @@ -54,6 +56,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -193,6 +196,101 @@ class MemoryBackendModuleTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueryOperators() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_SHAPE); + QSession session = new QSession(); + + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(session); + insertInput.setTableName(table.getName()); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("name", "Square").withValue("date", LocalDate.of(1980, Month.MAY, 31)), + new QRecord().withValue("id", 2).withValue("name", "Triangle").withValue("date", LocalDate.of(1999, Month.DECEMBER, 31)) + )); + new InsertAction().execute(insertInput); + + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 2))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.IN, List.of(2, 3))).size()); + + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.NOT_IN, List.of(3, 4))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.NOT_IN, List.of(2, 3))).size()); + + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.EQUALS, List.of("Square"))).size()); + assertEquals("Square", queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.EQUALS, List.of("Square"))).get(0).getValue("name")); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("notAFieldSoNull", QCriteriaOperator.EQUALS, List.of("Square"))).size()); + + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.NOT_EQUALS, List.of("notFound"))).size()); + assertEquals("Square", queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.NOT_EQUALS, List.of("Triangle"))).get(0).getValue("name")); + + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.CONTAINS, List.of("ria"))).size()); + assertEquals("Triangle", queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.CONTAINS, List.of("ria"))).get(0).getValue("name")); + + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.NOT_CONTAINS, List.of("notFound"))).size()); + assertEquals("Square", queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.NOT_CONTAINS, List.of("ria"))).get(0).getValue("name")); + + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.CONTAINS, List.of("ria")))); + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.CONTAINS, List.of(1)))); + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.CONTAINS, List.of()))); + + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN, List.of(LocalDate.of(2022, Month.SEPTEMBER, 1)))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN, List.of(LocalDate.of(1990, Month.JANUARY, 1)))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN, List.of(LocalDate.of(1970, Month.JANUARY, 1)))).size()); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, List.of(2))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, List.of(1))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, List.of(0))).size()); + + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(LocalDate.of(2022, Month.SEPTEMBER, 1)))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(LocalDate.of(1990, Month.JANUARY, 1)))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(LocalDate.of(1970, Month.JANUARY, 1)))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(LocalDate.of(1980, Month.MAY, 31)))).size()); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(3))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(2))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(1))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of(0))).size()); + + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN, List.of(LocalDate.of(2022, Month.SEPTEMBER, 1)))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN, List.of(LocalDate.of(1990, Month.JANUARY, 1)))).size()); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN, List.of(LocalDate.of(1970, Month.JANUARY, 1)))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN, List.of(3))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN, List.of(2))).size()); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN, List.of(1))).size()); + + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(LocalDate.of(2022, Month.SEPTEMBER, 1)))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(LocalDate.of(1990, Month.JANUARY, 1)))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(LocalDate.of(1980, Month.MAY, 31)))).size()); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("date", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(LocalDate.of(1970, Month.JANUARY, 1)))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(3))).size()); + assertEquals(2, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(2))).size()); + assertEquals(1, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(1))).size()); + assertEquals(0, queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(0))).size()); + + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, List.of()))); + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, List.of()))); + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN, List.of()))); + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of()))); + assertThrows(QException.class, () -> queryShapes(qInstance, table, session, new QFilterCriteria("name", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of("Bob")))); + } + + + + private List queryShapes(QInstance qInstance, QTableMetaData table, QSession session, QFilterCriteria criteria) throws QException + { + QueryInput queryInput = new QueryInput(qInstance); + queryInput.setSession(session); + queryInput.setTableName(table.getName()); + queryInput.setFilter(new QQueryFilter().withCriteria(criteria)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + return queryOutput.getRecords(); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index b762d42f..8f444850 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -24,11 +24,13 @@ package com.kingsrook.qqq.backend.core.utils; import java.io.Serializable; import java.time.LocalDate; +import java.time.Month; import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarChart; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.GetAgeStatistics; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; @@ -36,6 +38,8 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; @@ -67,10 +71,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -105,6 +109,7 @@ public class TestUtils public static final String PROCESS_NAME_GREET_PEOPLE = "greet"; public static final String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive"; + public static final String PROCESS_NAME_INCREASE_BIRTHDATE = "increaseBirthdate"; public static final String PROCESS_NAME_ADD_TO_PEOPLES_AGE = "addToPeoplesAge"; public static final String TABLE_NAME_PERSON_FILE = "personFile"; public static final String TABLE_NAME_PERSON_MEMORY = "personMemory"; @@ -146,6 +151,7 @@ public class TestUtils qInstance.addProcess(defineProcessAddToPeoplesAge()); qInstance.addProcess(new BasicETLProcess().defineProcessMetaData()); qInstance.addProcess(new StreamedETLProcess().defineProcessMetaData()); + qInstance.addProcess(defineProcessIncreasePersonBirthdate()); qInstance.addAutomationProvider(definePollingAutomationProvider()); @@ -170,6 +176,72 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + private static QProcessMetaData defineProcessIncreasePersonBirthdate() + { + return new QProcessMetaData() + .withName(PROCESS_NAME_INCREASE_BIRTHDATE) + .withTableName(TABLE_NAME_PERSON_MEMORY) + + .addStep(new QFrontendStepMetaData() + .withName("preview") + ) + + .addStep(new QBackendStepMetaData() + .withName("doWork") + .withCode(new QCodeReference(IncreaseBirthdateStep.class)) + .withInputData(new QFunctionInputMetaData() + .withRecordListMetaData(new QRecordListMetaData().withTableName(TABLE_NAME_PERSON_MEMORY))) + .withOutputMetaData(new QFunctionOutputMetaData() + .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING).withDefaultValue("Success!")))) + ) + + .addStep(new QFrontendStepMetaData() + .withName("results") + .withFormField(new QFieldMetaData("outputMessage", QFieldType.STRING)) + ); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class IncreaseBirthdateStep implements BackendStep + { + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + List recordsToUpdate = new ArrayList<>(); + for(QRecord record : runBackendStepInput.getRecords()) + { + LocalDate birthDate = record.getValueLocalDate("birthDate"); + + if(birthDate != null && birthDate.getYear() < 1900) + { + recordsToUpdate.add(new QRecord() + .withValue("id", record.getValue("id")) + .withValue("birthDate", birthDate.withYear(1900)) + ); + } + } + + UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance()); + updateInput.setSession(runBackendStepInput.getSession()); + updateInput.setTableName(TABLE_NAME_PERSON_MEMORY); + updateInput.setRecords(recordsToUpdate); + new UpdateAction().execute(updateInput); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -392,6 +464,16 @@ public class TestUtils *******************************************************************************/ public static QTableMetaData definePersonMemoryTable() { + ///////////////////////////////////////////////////////////////////////////// + // the checkAge automation will only run on persons younger than this date // + ///////////////////////////////////////////////////////////////////////////// + LocalDate youngPersonLimitDate = LocalDate.now().minusYears(18); + + ///////////////////////////////////////////////////////////////////////////////////// + // the increaseBirthdate automation will only run on persons born before this date // + ///////////////////////////////////////////////////////////////////////////////////// + LocalDate increaseBirthdateLimitDate = LocalDate.of(1900, Month.JANUARY, 1); + return (new QTableMetaData() .withName(TABLE_NAME_PERSON_MEMORY) .withBackendName(MEMORY_BACKEND_NAME) @@ -403,12 +485,19 @@ public class TestUtils .withAction(new TableAutomationAction() .withName("checkAgeOnInsert") .withTriggerEvent(TriggerEvent.POST_INSERT) + .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, List.of(youngPersonLimitDate)))) .withCodeReference(new QCodeReference(CheckAge.class)) ) + .withAction(new TableAutomationAction() + .withName("increaseBirthdate") + .withTriggerEvent(TriggerEvent.POST_INSERT) + .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, List.of(increaseBirthdateLimitDate)))) + .withProcessName(PROCESS_NAME_INCREASE_BIRTHDATE) + ) .withAction(new TableAutomationAction() .withName("logOnUpdatePerFilter") .withTriggerEvent(TriggerEvent.POST_UPDATE) - .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Darin")))) + .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.CONTAINS, List.of("Darin")))) .withCodeReference(new QCodeReference(LogPersonUpdate.class)) ) ); @@ -437,7 +526,6 @@ public class TestUtils LocalDate birthDate = record.getValueLocalDate("birthDate"); if(birthDate != null && birthDate.isAfter(limitDate)) { - LOG.info("Person [" + record.getValueInteger("id") + "] is a minor - updating their firstName to state such."); recordsToUpdate.add(new QRecord() .withValue("id", record.getValue("id")) .withValue("firstName", record.getValueString("firstName") + SUFFIX_FOR_MINORS) From 9a8b49f1a7e6afc6dbfca5f234e2ddfbd9e48ce4 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 5 Sep 2022 09:47:43 -0500 Subject: [PATCH 22/28] Feedback from code reviews --- checkstyle.xml => checkstyle/config.xml | 4 + checkstyle/license.txt | 20 +++ pom.xml | 3 +- .../core/actions/async/AsyncJobCallback.java | 11 ++ .../actions/async/AsyncRecordPipeLoop.java | 21 +++ .../actions/automation/AutomationStatus.java | 21 +++ .../automation/RecordAutomationHandler.java | 21 +++ .../RecordAutomationStatusUpdater.java | 38 ++++-- .../polling/PollingAutomationExecutor.java | 24 +++- .../polling/PollingAutomationRunner.java | 125 ++++++++++++------ .../actions/customizers/TableCustomizer.java | 21 +++ .../actions/customizers/TableCustomizers.java | 21 +++ .../dashboard/AbstractWidgetRenderer.java | 21 +++ .../dashboard/QuickSightChartRenderer.java | 21 +++ .../actions/dashboard/WidgetDataLoader.java | 21 +++ .../actions/interfaces/QActionInterface.java | 21 +++ .../actions/processes/RunProcessAction.java | 5 +- .../core/actions/reporting/RecordPipe.java | 4 +- .../core/instances/QInstanceValidator.java | 5 +- .../actions/processes/ProcessSummaryLine.java | 21 +++ .../processes/RunBackendStepInput.java | 4 +- .../core/model/actions/processes/Status.java | 23 +++- .../automation/RecordAutomationInput.java | 64 +++------ .../model/dashboard/widgets/BarChart.java | 21 +++ .../core/model/dashboard/widgets/QWidget.java | 21 +++ .../dashboard/widgets/QuickSightChart.java | 21 +++ .../qqq/backend/core/model/data/QRecord.java | 17 ++- .../PollingAutomationProviderMetaData.java | 21 +++ .../QAutomationProviderMetaData.java | 21 +++ .../metadata/dashboard/QWidgetMetaData.java | 21 +++ .../dashboard/QWidgetMetaDataInterface.java | 21 +++ .../dashboard/QuickSightChartMetaData.java | 21 +++ .../PVSValueFormatAndFields.java | 21 +++ .../automation/AutomationStatusTracking.java | 21 +++ .../AutomationStatusTrackingType.java | 21 +++ .../automation/QTableAutomationDetails.java | 21 +++ .../automation/TableAutomationAction.java | 21 +++ .../tables/automation/TriggerEvent.java | 21 +++ .../implementations/mock/MockQueryAction.java | 11 ++ .../AbstractExtractStep.java | 21 +++ .../AbstractLoadStep.java | 23 +++- .../AbstractTransformStep.java | 23 +++- .../BaseStreamedETLStep.java | 47 ++++++- .../ExtractViaQueryStep.java | 23 ++++ .../LoadViaInsertStep.java | 21 +++ .../LoadViaUpdateStep.java | 21 +++ .../ProcessSummaryProviderInterface.java | 25 +++- .../StreamedETLExecuteStep.java | 13 +- .../StreamedETLPreviewStep.java | 27 +--- .../StreamedETLValidateStep.java | 24 +--- .../StreamedETLWithFrontendProcess.java | 6 +- .../PollingAutomationExecutorTest.java | 35 +++-- .../polling/PollingAutomationRunnerTest.java | 119 ++++++++++++++--- .../PersonsByCreateDateBarChart.java | 21 +++ .../dashboard/WidgetDataLoaderTest.java | 21 +++ .../actions/processes/RunProcessTest.java | 6 +- .../instances/QInstanceValidatorTest.java | 25 +--- .../StreamedETLWithFrontendProcessTest.java | 50 +++++-- .../actions/FilesystemTableCustomizers.java | 21 +++ .../javalin/PersonsByCreateDateBarChart.java | 21 +++ .../widgets/PersonsByCreateDateBarChart.java | 21 +++ .../clonepeople/ClonePeopleTransformStep.java | 21 +++ .../sampleapp/SampleJavalinServerTest.java | 21 +++ .../PersonsByCreateDateBarChartTest.java | 21 +++ .../ClonePeopleTransformStepTest.java | 21 +++ 65 files changed, 1337 insertions(+), 223 deletions(-) rename checkstyle.xml => checkstyle/config.xml (98%) create mode 100644 checkstyle/license.txt diff --git a/checkstyle.xml b/checkstyle/config.xml similarity index 98% rename from checkstyle.xml rename to checkstyle/config.xml index f5e7412d..2e563475 100644 --- a/checkstyle.xml +++ b/checkstyle/config.xml @@ -261,5 +261,9 @@ + + + + diff --git a/checkstyle/license.txt b/checkstyle/license.txt new file mode 100644 index 00000000..299e0368 --- /dev/null +++ b/checkstyle/license.txt @@ -0,0 +1,20 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ diff --git a/pom.xml b/pom.xml index e551b021..1ba866ed 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,8 @@ validate validate - checkstyle.xml + checkstyle/config.xml + checkstyle/license.txt UTF-8 true diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobCallback.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobCallback.java index ec9ff1de..4aeefb90 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobCallback.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobCallback.java @@ -50,6 +50,17 @@ public class AsyncJobCallback + /******************************************************************************* + ** Setter for asyncJobStatus + ** + *******************************************************************************/ + public void setAsyncJobStatus(AsyncJobStatus asyncJobStatus) + { + this.asyncJobStatus = asyncJobStatus; + } + + + /******************************************************************************* ** Update the message *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java index d41782bd..0a1273dc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.async; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java index ca0b1158..0c73523c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.automation; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationHandler.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationHandler.java index 74d76fa6..9358b58e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationHandler.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationHandler.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.automation; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java index d31b7297..5b6ec879 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.automation; @@ -46,6 +67,13 @@ public class RecordAutomationStatusUpdater automationStatus = AutomationStatus.OK; } + /////////////////////////////////////////////////////////////////////////////////////////////////// + // In case an automation is running, and it updates records - don't let those records be marked // + // as PENDING_UPDATE_AUTOMATIONS... this is meant to avoid having a record's automation update // + // itself, and then continue to do so in a loop (infinitely). // + // BUT - shouldn't this be allowed to update OTHER records to be pending updates? It seems like // + // yes - so -that'll probably be a bug to fix at some point in the future todo // + /////////////////////////////////////////////////////////////////////////////////////////////////// if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS)) { Exception e = new Exception(); @@ -86,17 +114,11 @@ public class RecordAutomationStatusUpdater if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS)) { - if(tableActions.stream().noneMatch(a -> TriggerEvent.POST_INSERT.equals(a.getTriggerEvent()))) - { - return (true); - } + return tableActions.stream().noneMatch(a -> TriggerEvent.POST_INSERT.equals(a.getTriggerEvent())); } else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS)) { - if(tableActions.stream().noneMatch(a -> TriggerEvent.POST_UPDATE.equals(a.getTriggerEvent()))) - { - return (true); - } + return tableActions.stream().noneMatch(a -> TriggerEvent.POST_UPDATE.equals(a.getTriggerEvent())); } return (false); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java index 205a16e3..2837577c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.automation.polling; @@ -82,8 +103,7 @@ public class PollingAutomationExecutor *******************************************************************************/ public void stopAsync() { - Runnable stopper = this::stop; - stopper.run(); + new Thread(this::stop).start(); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java index 06f31946..896827f2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java @@ -1,8 +1,30 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.automation.polling; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -14,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; @@ -28,14 +51,14 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent; import com.kingsrook.qqq.backend.core.model.session.QSession; -import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.apache.commons.lang.NotImplementedException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -57,6 +80,23 @@ class PollingAutomationRunner implements Runnable private Map> tableInsertActions = new HashMap<>(); private Map> tableUpdateActions = new HashMap<>(); + private Map>> tableActions = new HashMap<>(); + + private static Map triggerEventAutomationStatusMap = Map.of( + TriggerEvent.POST_INSERT, AutomationStatus.PENDING_INSERT_AUTOMATIONS, + TriggerEvent.POST_UPDATE, AutomationStatus.PENDING_UPDATE_AUTOMATIONS + ); + + private static Map pendingToRunningStatusMap = Map.of( + AutomationStatus.PENDING_INSERT_AUTOMATIONS, AutomationStatus.RUNNING_INSERT_AUTOMATIONS, + AutomationStatus.PENDING_UPDATE_AUTOMATIONS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS + ); + + private static Map pendingToFailedStatusMap = Map.of( + AutomationStatus.PENDING_INSERT_AUTOMATIONS, AutomationStatus.FAILED_INSERT_AUTOMATIONS, + AutomationStatus.PENDING_UPDATE_AUTOMATIONS, AutomationStatus.FAILED_UPDATE_AUTOMATIONS + ); + /******************************************************************************* @@ -83,16 +123,10 @@ class PollingAutomationRunner implements Runnable /////////////////////////////////////////////////////////////////////////// for(TableAutomationAction action : table.getAutomationDetails().getActions()) { - if(TriggerEvent.POST_INSERT.equals(action.getTriggerEvent())) - { - tableInsertActions.putIfAbsent(table.getName(), new ArrayList<>()); - tableInsertActions.get(table.getName()).add(action); - } - else if(TriggerEvent.POST_UPDATE.equals(action.getTriggerEvent())) - { - tableUpdateActions.putIfAbsent(table.getName(), new ArrayList<>()); - tableUpdateActions.get(table.getName()).add(action); - } + AutomationStatus automationStatus = triggerEventAutomationStatusMap.get(action.getTriggerEvent()); + tableActions.putIfAbsent(table.getName(), new HashMap<>()); + tableActions.get(table.getName()).putIfAbsent(automationStatus, new ArrayList<>()); + tableActions.get(table.getName()).get(automationStatus).add(action); } ////////////////////////////// @@ -143,8 +177,8 @@ class PollingAutomationRunner implements Runnable private void processTable(QTableMetaData table) throws QException { QSession session = sessionSupplier != null ? sessionSupplier.get() : new QSession(); - processTableInsertOrUpdate(table, session, true); - processTableInsertOrUpdate(table, session, false); + processTableInsertOrUpdate(table, session, AutomationStatus.PENDING_INSERT_AUTOMATIONS); + processTableInsertOrUpdate(table, session, AutomationStatus.PENDING_UPDATE_AUTOMATIONS); } @@ -152,10 +186,11 @@ class PollingAutomationRunner implements Runnable /******************************************************************************* ** Query for and process records that have a PENDING_INSERT or PENDING_UPDATE status on a given table. *******************************************************************************/ - private void processTableInsertOrUpdate(QTableMetaData table, QSession session, boolean isInsert) throws QException + private void processTableInsertOrUpdate(QTableMetaData table, QSession session, AutomationStatus automationStatus) throws QException { - AutomationStatus automationStatus = isInsert ? AutomationStatus.PENDING_INSERT_AUTOMATIONS : AutomationStatus.PENDING_UPDATE_AUTOMATIONS; - List actions = (isInsert ? tableInsertActions : tableUpdateActions).get(table.getName()); + List actions = tableActions + .getOrDefault(table.getName(), Collections.emptyMap()) + .getOrDefault(automationStatus, Collections.emptyList()); if(CollectionUtils.nullSafeIsEmpty(actions)) { return; @@ -166,20 +201,30 @@ class PollingAutomationRunner implements Runnable /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // run an async-pipe loop - that will query for records in PENDING - put them in a pipe - then apply actions to them // /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - RecordPipe recordPipe = new RecordPipe(); + RecordPipe recordPipe = new RecordPipe(); AsyncRecordPipeLoop asyncRecordPipeLoop = new AsyncRecordPipeLoop(); - asyncRecordPipeLoop.run("PollingAutomationRunner>Query>" + (isInsert ? "insert" : "update"), null, recordPipe, (status) -> + asyncRecordPipeLoop.run("PollingAutomationRunner>Query>" + automationStatus, null, recordPipe, (status) -> { QueryInput queryInput = new QueryInput(instance); queryInput.setSession(session); queryInput.setTableName(table.getName()); - queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(table.getAutomationDetails().getStatusTracking().getFieldName(), QCriteriaOperator.IN, List.of(automationStatus.getId())))); + + AutomationStatusTrackingType statusTrackingType = table.getAutomationDetails().getStatusTracking().getType(); + if(AutomationStatusTrackingType.FIELD_IN_TABLE.equals(statusTrackingType)) + { + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(table.getAutomationDetails().getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId())))); + } + else + { + throw (new NotImplementedException("Automation Status Tracking type [" + statusTrackingType + "] is not yet implemented in here.")); + } + queryInput.setRecordPipe(recordPipe); return (new QueryAction().execute(queryInput)); }, () -> { List records = recordPipe.consumeAvailableRecords(); - applyActionsToRecords(session, table, records, actions, isInsert); + applyActionsToRecords(session, table, records, actions, automationStatus); return (records.size()); } ); @@ -189,9 +234,10 @@ class PollingAutomationRunner implements Runnable /******************************************************************************* ** For a set of records that were found to be in a PENDING state - run all the - ** table's actions against them. + ** table's actions against them - IF they are found to match the action's filter + ** (assuming it has one - if it doesn't, then all records match). *******************************************************************************/ - private void applyActionsToRecords(QSession session, QTableMetaData table, List records, List actions, boolean isInsert) throws QException + private void applyActionsToRecords(QSession session, QTableMetaData table, List records, List actions, AutomationStatus automationStatus) throws QException { if(CollectionUtils.nullSafeIsEmpty(records)) { @@ -201,7 +247,7 @@ class PollingAutomationRunner implements Runnable /////////////////////////////////////////////////// // mark the records as RUNNING their automations // /////////////////////////////////////////////////// - RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, isInsert ? AutomationStatus.RUNNING_INSERT_AUTOMATIONS : AutomationStatus.RUNNING_UPDATE_AUTOMATIONS); + RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToRunningStatusMap.get(automationStatus)); /////////////////////////////////////////////////////////////////////////////////////////////////////////////// // foreach action - run it against the records (but only if they match the action's filter, if there is one) // @@ -215,7 +261,7 @@ class PollingAutomationRunner implements Runnable // note - this method - will re-query the objects, so we should have confidence that their data is fresh... // ////////////////////////////////////////////////////////////////////////////////////////////////////////////// List matchingQRecords = getRecordsMatchingActionFilter(session, table, records, action); - LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action); + LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action); if(CollectionUtils.nullSafeHasContents(matchingQRecords)) { LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action); @@ -234,7 +280,7 @@ class PollingAutomationRunner implements Runnable //////////////////////////////////////// if(anyActionsFailed) { - RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.FAILED_UPDATE_AUTOMATIONS); + RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToFailedStatusMap.get(automationStatus)); } else { @@ -302,25 +348,24 @@ class PollingAutomationRunner implements Runnable { if(StringUtils.hasContent(action.getProcessName())) { + ///////////////////////////////////////////////////////////////////////////////////////// + // if the action has a process associated with it - run that process. // + // tell it to SKIP frontend steps. // + // give the process a callback w/ a query filter that has the p-keys of these records. // + ///////////////////////////////////////////////////////////////////////////////////////// RunProcessInput runProcessInput = new RunProcessInput(instance); runProcessInput.setSession(session); runProcessInput.setProcessName(action.getProcessName()); runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); - - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // kinda hacky - if we see that this process has an input field of a given name, then put a filter in there to find these records... // - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QProcessMetaData process = instance.getProcess(action.getProcessName()); - if(process.getInputFields().stream().anyMatch(f -> f.getName().equals(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER))) + runProcessInput.setCallback(new QProcessCallback() { - List recordIds = records.stream().map(r -> r.getValueInteger(table.getPrimaryKeyField())).collect(Collectors.toList()); - QQueryFilter queryFilter = new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds)); - runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, queryFilter); - } - else - { - runProcessInput.setRecords(records); - } + @Override + public QQueryFilter getQueryFilter() + { + List recordIds = records.stream().map(r -> r.getValueInteger(table.getPrimaryKeyField())).collect(Collectors.toList()); + return (new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds))); + } + }); RunProcessAction runProcessAction = new RunProcessAction(); RunProcessOutput runProcessOutput = runProcessAction.execute(runProcessInput); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizer.java index 9367a0c1..f61bf8ce 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizer.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.customizers; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java index ef5f77b1..633901ee 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.customizers; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java index 7919b928..93370857 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.dashboard; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java index 74687c75..5ce05407 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.dashboard; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java index de346886..83d2676c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.dashboard; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QActionInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QActionInterface.java index cac3f87f..88bc436a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QActionInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QActionInterface.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.interfaces; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index 39a9fc51..6b7ce8ed 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -140,7 +140,7 @@ public class RunProcessAction /////////////////////// // Run backend steps // /////////////////////// - LOG.info("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]"); + LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]"); runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState); } else @@ -191,7 +191,8 @@ public class RunProcessAction if(runProcessInput.getStartAfterStep() == null) { /////////////////////////////////////////////////////////////////////////////////// - // this is fine - it means it's our first time running in the backend. // + // This condition (no state in state-provider, and no start-after-step) means // + // that we're starting a new process! Init the process state here, then // // Go ahead and store the state that we have (e.g., w/ initial records & values) // /////////////////////////////////////////////////////////////////////////////////// ProcessState processState = runProcessInput.getProcessState(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java index 548e8869..218e920a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java @@ -72,7 +72,7 @@ public class RecordPipe /******************************************************************************* - ** Add a record to the pipe + ** Add a record to the pipe. Will block if the pipe is full. Will noop if pipe is terminated. *******************************************************************************/ public void addRecord(QRecord record) { @@ -129,7 +129,7 @@ public class RecordPipe /******************************************************************************* - ** Add a list of records to the pipe + ** Add a list of records to the pipe. Will block if the pipe is full. Will noop if pipe is terminated. *******************************************************************************/ public void addRecords(List records) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 3d5a5839..f1148d43 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -291,7 +291,10 @@ public class QInstanceValidator { if(statusTracking.getType().equals(AutomationStatusTrackingType.FIELD_IN_TABLE)) { - assertCondition(StringUtils.hasContent(statusTracking.getFieldName()), prefix + "statusTracking of type fieldInTable is missing its fieldName"); + if(assertCondition(StringUtils.hasContent(statusTracking.getFieldName()), prefix + "statusTracking of type fieldInTable is missing its fieldName")) + { + assertNoException(() -> table.getField(statusTracking.getFieldName()), prefix + "statusTracking field is not a defined field on this table."); + } } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java index 21c5db44..7a21349f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.actions.processes; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index dd85679b..cab801b9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -371,10 +371,10 @@ public class RunBackendStepInput extends AbstractActionInput /******************************************************************************* - ** Getter for a single field's value as a primitive boolean + ** Getter for a single field's value as a primitive boolean - with null => false. ** *******************************************************************************/ - public boolean getValue_boolean(String fieldName) + public boolean getValuePrimitiveBoolean(String fieldName) { Boolean valueAsBoolean = ValueUtils.getValueAsBoolean(getValue(fieldName)); return (valueAsBoolean != null && valueAsBoolean); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java index 60eb77ad..ee920b8f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java @@ -1,8 +1,29 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.actions.processes; /******************************************************************************* - ** Simple status enum - initially for statusesqqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java in process status lines. + ** Simple status enum - initially for statuses in process status lines. *******************************************************************************/ public enum Status { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/automation/RecordAutomationInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/automation/RecordAutomationInput.java index d3cd1c4d..17042c3a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/automation/RecordAutomationInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/automation/RecordAutomationInput.java @@ -1,7 +1,27 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.automation; -import java.io.Serializable; import java.util.List; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -15,13 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAuto public class RecordAutomationInput extends AbstractTableActionInput { private TableAutomationAction action; - - //////////////////////////////////////////// - // todo - why both? pick one? or don't? // - // maybe - if recordList is null and primaryKeyList isn't, then do the record query in here? - //////////////////////////////////////////// - private List recordList; - private List primaryKeyList; + private List recordList; @@ -101,38 +115,4 @@ public class RecordAutomationInput extends AbstractTableActionInput return (this); } - - - /******************************************************************************* - ** Getter for primaryKeyList - ** - *******************************************************************************/ - public List getPrimaryKeyList() - { - return primaryKeyList; - } - - - - /******************************************************************************* - ** Setter for primaryKeyList - ** - *******************************************************************************/ - public void setPrimaryKeyList(List primaryKeyList) - { - this.primaryKeyList = primaryKeyList; - } - - - - /******************************************************************************* - ** Fluent setter for primaryKeyList - ** - *******************************************************************************/ - public RecordAutomationInput withPrimaryKeyList(List primaryKeyList) - { - this.primaryKeyList = primaryKeyList; - return (this); - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java index 2d8e8667..3fd0b73f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.dashboard.widgets; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java index 8c76fc52..81f1d400 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.dashboard.widgets; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java index 9cccd508..e5561cac 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.dashboard.widgets; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index 4206c97b..5e3772e1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -90,7 +90,6 @@ public class QRecord implements Serializable ** Copy constructor. ** *******************************************************************************/ - @SuppressWarnings("unchecked") public QRecord(QRecord record) { this.tableName = record.tableName; @@ -118,8 +117,8 @@ public class QRecord implements Serializable /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings({ "rawtypes", "unchecked" }) - private Map doDeepCopy(Map map) + @SuppressWarnings({ "unchecked" }) + private Map doDeepCopy(Map map) { if(map == null) { @@ -128,10 +127,10 @@ public class QRecord implements Serializable if(map instanceof Serializable serializableMap) { - return (Map) SerializationUtils.clone(serializableMap); + return (Map) SerializationUtils.clone(serializableMap); } - return (new LinkedHashMap(map)); + return (new LinkedHashMap<>(map)); } @@ -139,8 +138,8 @@ public class QRecord implements Serializable /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings({ "rawtypes", "unchecked" }) - private List doDeepCopy(List list) + @SuppressWarnings({ "unchecked" }) + private List doDeepCopy(List list) { if(list == null) { @@ -149,10 +148,10 @@ public class QRecord implements Serializable if(list instanceof Serializable serializableList) { - return (List) SerializationUtils.clone(serializableList); + return (List) SerializationUtils.clone(serializableList); } - return (new ArrayList(list)); + return (new ArrayList<>(list)); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/PollingAutomationProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/PollingAutomationProviderMetaData.java index e5bb0283..4acf731a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/PollingAutomationProviderMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/PollingAutomationProviderMetaData.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.metadata.automation; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java index ae8c8f00..51d1cda0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.metadata.automation; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java index 2a9594b7..3bf01462 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.metadata.dashboard; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java index 771c8d33..162fc56a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.metadata.dashboard; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java index 114804a4..c83ff288 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.metadata.dashboard; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PVSValueFormatAndFields.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PVSValueFormatAndFields.java index f8ea0eb9..fea83756 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PVSValueFormatAndFields.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PVSValueFormatAndFields.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.metadata.possiblevalues; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTracking.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTracking.java index 7d504465..e9f5d69e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTracking.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTracking.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.metadata.tables.automation; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTrackingType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTrackingType.java index 6d67f046..35e1446b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTrackingType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTrackingType.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.metadata.tables.automation; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java index fadfc06d..3be1f343 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.metadata.tables.automation; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TableAutomationAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TableAutomationAction.java index 295e4bdc..d3e008e1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TableAutomationAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TableAutomationAction.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.metadata.tables.automation; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TriggerEvent.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TriggerEvent.java index 4b64ebe2..7dac0122 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TriggerEvent.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TriggerEvent.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.model.metadata.tables.automation; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java index ff3761f7..ec0589fc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java @@ -35,6 +35,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -43,6 +45,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; *******************************************************************************/ public class MockQueryAction implements QueryInterface { + private static final Logger LOG = LogManager.getLogger(MockQueryAction.class); + + /******************************************************************************* ** @@ -68,6 +73,12 @@ public class MockQueryAction implements QueryInterface } queryOutput.addRecord(record); + + if(queryInput.getAsyncJobCallback().wasCancelRequested()) + { + LOG.info("Breaking query job, as requested."); + break; + } } return (queryOutput); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java index 5d4b5c2a..ea1dbc58 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java index 08381b0d..55b82768 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; @@ -19,7 +40,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** should be written to the outputRecordPage. That is to say, DO NOT use the ** recordList in the step input/output objects. ** - ** Also - use the transaction member variable - though be aware, it + ** Also - use the transaction member variable!!! *******************************************************************************/ public abstract class AbstractLoadStep implements BackendStep { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java index 2f9c416e..ffe4e7ef 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; @@ -17,7 +38,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; ** should be written to the outputRecordPage. That is to say, DO NOT use the ** recordList in the step input/output objects. *******************************************************************************/ -public abstract class AbstractTransformStep implements BackendStep +public abstract class AbstractTransformStep implements BackendStep, ProcessSummaryProviderInterface { private List inputRecordPage = new ArrayList<>(); private List outputRecordPage = new ArrayList<>(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java index c860176d..2dd6c0ef 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java @@ -1,14 +1,39 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; +import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -16,6 +41,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; *******************************************************************************/ public class BaseStreamedETLStep { + private static final Logger LOG = LogManager.getLogger(BaseStreamedETLStep.class); + protected static final int PROCESS_OUTPUT_RECORD_LIST_LIMIT = 20; @@ -58,8 +85,8 @@ public class BaseStreamedETLStep *******************************************************************************/ protected void updateRecordsWithDisplayValuesAndPossibleValues(RunBackendStepInput input, List list) { - String destinationTable = input.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE); - QTableMetaData table = input.getInstance().getTable(destinationTable); + String destinationTable = input.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE); + QTableMetaData table = input.getInstance().getTable(destinationTable); if(table != null && list != null) { @@ -70,4 +97,20 @@ public class BaseStreamedETLStep qPossibleValueTranslator.translatePossibleValuesInRecords(input.getTable(), list); } } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void moveReviewStepAfterValidateStep(RunBackendStepOutput runBackendStepOutput) + { + LOG.info("Skipping to validation step"); + ArrayList stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); + LOG.debug("Step list pre: " + stepList); + stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)); + stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); + runBackendStepOutput.getProcessState().setStepList(stepList); + LOG.debug("Step list post: " + stepList); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java index 0e1b81ae..aa94712a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; @@ -12,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; @@ -42,6 +64,7 @@ public class ExtractViaQueryStep extends AbstractExtractStep queryInput.setFilter(getQueryFilter(runBackendStepInput)); queryInput.setRecordPipe(getRecordPipe()); queryInput.setLimit(getLimit()); + queryInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); new QueryAction().execute(queryInput); /////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java index 10c0567f..fe0f1a29 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java index 6c34b623..69eaffd1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ProcessSummaryProviderInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ProcessSummaryProviderInterface.java index a4b74fe4..aa09fd24 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ProcessSummaryProviderInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ProcessSummaryProviderInterface.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; @@ -6,13 +27,13 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine /******************************************************************************* - ** + ** Interface for a class that can proivate a ProcessSummary - a list of Process Summary Lines *******************************************************************************/ public interface ProcessSummaryProviderInterface { /******************************************************************************* - ** + ** Note - object needs to be serializable, and List isn't... so, use ArrayList? *******************************************************************************/ ArrayList getProcessSummary(boolean isForResultScreen); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java index aa88dc8d..2c5f413f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java @@ -68,6 +68,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe AbstractLoadStep loadStep = getLoadStep(runBackendStepInput); transformStep.preRun(runBackendStepInput, runBackendStepOutput); + loadStep.preRun(runBackendStepInput, runBackendStepOutput); transaction = loadStep.openTransaction(runBackendStepInput); loadStep.setTransaction(transaction); @@ -86,14 +87,10 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe updateRecordsWithDisplayValuesAndPossibleValues(runBackendStepInput, loadedRecordList); runBackendStepOutput.setRecords(loadedRecordList); - if(transformStep instanceof ProcessSummaryProviderInterface processSummaryProvider) - { - ////////////////////////////////////////////////////////////////////////////////////////////// - // get the process summary from the ... transform step? the load step? each knows some... // - // TODO!! // - ////////////////////////////////////////////////////////////////////////////////////////////// - runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummaryProvider.getProcessSummary(true)); - } + //////////////////////////////////////////////////////////////////////////////////////////////////// + // get the process summary from the ... transform step? the load step? each knows some... todo? // + //////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, transformStep.getProcessSummary(true)); transformStep.postRun(runBackendStepInput, runBackendStepOutput); loadStep.postRun(runBackendStepInput, runBackendStepOutput); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java index 59eb69c1..74367ec4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -57,11 +57,12 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe ////////////////////////////////////////////////////////////////////////////////////////////////////////// // if the do-full-validation flag has already been set, then do the validation step instead of this one // ////////////////////////////////////////////////////////////////////////////////////////////////////////// - boolean supportsFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION); - boolean doFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION); + boolean supportsFullValidation = runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION); + boolean doFullValidation = runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION); if(supportsFullValidation && doFullValidation) { - skipToValidateStep(runBackendStepOutput); + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + moveReviewStepAfterValidateStep(runBackendStepOutput); return; } @@ -87,7 +88,8 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe // todo - maybe some future version we do this - maybe based on a user-preference // if(supportsFullValidation && recordCount <= limit) // { - // skipToValidateStep(runBackendStepOutput); + // runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + // moveReviewStepAfterValidateStep(runBackendStepOutput); // return; // } @@ -104,6 +106,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe List previewRecordList = new ArrayList<>(); new AsyncRecordPipeLoop().run("StreamedETL>Preview>ExtractStep", PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> { + runBackendStepInput.setAsyncJobCallback(status); extractStep.run(runBackendStepInput, runBackendStepOutput); return (runBackendStepOutput); }, @@ -137,22 +140,6 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe - /******************************************************************************* - ** - *******************************************************************************/ - private void skipToValidateStep(RunBackendStepOutput runBackendStepOutput) - { - LOG.info("Skipping to validation step"); - runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); - ArrayList stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); - System.out.println("Step list pre: " + stepList); - stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)); - stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); - runBackendStepOutput.getProcessState().setStepList(stepList); - System.out.println("Step list post: " + stepList); - } - - /******************************************************************************* ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java index 58c3780c..adcd570b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java @@ -56,7 +56,7 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back ///////////////////////////////////////////////////////////////////// // check if we are supported in this process - if not, return noop // ///////////////////////////////////////////////////////////////////// - boolean supportsFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION); + boolean supportsFullValidation = runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION); if(!supportsFullValidation) { LOG.info("Process does not support validation, so skipping validation step"); @@ -66,22 +66,17 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back //////////////////////////////////////////////////////////////////////////////// // check if we've been requested to run in this process - if not, return noop // //////////////////////////////////////////////////////////////////////////////// - boolean doFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION); + boolean doFullValidation = runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION); if(!doFullValidation) { LOG.info("Not requested to do full validation, so skipping validation step"); return; } - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - // if we're proceeding with full validation, move the review step to be after validation in the step list // - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - ArrayList stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); - System.out.println("Step list pre: " + stepList); - stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)); - stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); - runBackendStepOutput.getProcessState().setStepList(stepList); - System.out.println("Step list post: " + stepList); + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we're proceeding with full validation, make sure the review step is after validation in the step list // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + moveReviewStepAfterValidateStep(runBackendStepOutput); ////////////////////////////////////////////////////////// // basically repeat the preview step, but with no limit // @@ -92,11 +87,6 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back extractStep.setRecordPipe(recordPipe); AbstractTransformStep transformStep = getTransformStep(runBackendStepInput); - if(!(transformStep instanceof ProcessSummaryProviderInterface processSummaryProvider)) - { - // todo - really? if this is required, then put it on the AbstractTransformStep class - throw (new QException("Transform Step " + transformStep.getClass().getName() + " does not implement ProcessSummaryProviderInterface.")); - } transformStep.preRun(runBackendStepInput, runBackendStepOutput); List previewRecordList = new ArrayList<>(); @@ -115,7 +105,7 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back ////////////////////////////////////////////////////// // get the process summary from the validation step // ////////////////////////////////////////////////////// - runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY, processSummaryProvider.getProcessSummary(false)); + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY, transformStep.getProcessSummary(false)); transformStep.postRun(runBackendStepInput, runBackendStepOutput); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 49b67354..7ae50735 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -41,10 +41,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; /******************************************************************************* ** Definition for Streamed ETL process that includes a frontend. ** - ** This process uses 2 backend steps, and 2 frontend steps, as follows: + ** This process uses 3 backend steps, and 2 frontend steps, as follows: ** - preview (backend) - does just a little work (limited # of rows), to give the ** user a preview of what the final result will be - e.g., some data to seed the review screen ** - review (frontend) - a review screen + ** - validate (backend) - optionally (per input on review screen), does like the preview step, + ** but on all records from the extract step. ** - execute (backend) - processes all the rows, does all the work. ** - result (frontend) - a result screen ** @@ -54,7 +56,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; ** - Transform - do whatever transformation is needed to the rows. Done on preview ** and execute. Always works with a "page" of records at a time. ** - Load - store the records into the backend, as appropriate. Always works - ** with a "page" of records at a time. + ** with a "page" of records at a time. Only called by execute step. *******************************************************************************/ public class StreamedETLWithFrontendProcess { diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java index a9cbb572..faa3b0b1 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.automation.polling; @@ -10,11 +31,8 @@ import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; -import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -76,19 +94,16 @@ class PollingAutomationExecutorTest ///////////////////////////////////////////////// // query for the records - assert their status // ///////////////////////////////////////////////// - QueryInput queryInput = new QueryInput(qInstance); - queryInput.setSession(new QSession()); - queryInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - assertEquals(2, queryOutput.getRecords().size()); + List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + assertEquals(2, records.size()); - Optional optionalPerson1 = queryOutput.getRecords().stream().filter(r -> r.getValueInteger("id") == 1).findFirst(); + Optional optionalPerson1 = records.stream().filter(r -> r.getValueInteger("id") == 1).findFirst(); assertThat(optionalPerson1).isPresent(); QRecord person1 = optionalPerson1.get(); assertThat(person1.getValueString("firstName")).isEqualTo("John"); assertThat(person1.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName())).isEqualTo(AutomationStatus.OK.getId()); - Optional optionalPerson2 = queryOutput.getRecords().stream().filter(r -> r.getValueInteger("id") == 2).findFirst(); + Optional optionalPerson2 = records.stream().filter(r -> r.getValueInteger("id") == 2).findFirst(); assertThat(optionalPerson2).isPresent(); QRecord person2 = optionalPerson2.get(); assertThat(person2.getValueString("firstName")).isEqualTo("Jim" + TestUtils.CheckAge.SUFFIX_FOR_MINORS); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunnerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunnerTest.java index 0b5da000..9061188e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunnerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunnerTest.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.automation.polling; @@ -14,8 +35,20 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcessTest; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -51,15 +84,15 @@ class PollingAutomationRunnerTest QInstance qInstance = TestUtils.defineInstance(); PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, TestUtils.POLLING_AUTOMATION, null); - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // insert 2 person records, one who should be both updated by the insert action, and should be logged by logger-on-update automation // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // insert 2 person records, both updated by the insert action, and 1 logged by logger-on-update automation // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// InsertInput insertInput = new InsertInput(qInstance); insertInput.setSession(new QSession()); insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); insertInput.setRecords(List.of( new QRecord().withValue("id", 1).withValue("firstName", "Tim").withValue("birthDate", LocalDate.now()), - new QRecord().withValue("id", 2).withValue("firstName", "Darin") + new QRecord().withValue("id", 2).withValue("firstName", "Darin").withValue("birthDate", LocalDate.now()) )); new InsertAction().execute(insertInput); assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS); @@ -72,15 +105,11 @@ class PollingAutomationRunnerTest assertThat(TestUtils.LogPersonUpdate.updatedIds).isNullOrEmpty(); assertAllRecordsAutomationStatus(AutomationStatus.OK); - //////////////////////////////////////////// - // make sure the minor person was updated // - //////////////////////////////////////////// - Optional updatedMinorRecord = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY).stream().filter(r -> r.getValueInteger("id").equals(1)).findFirst(); - assertThat(updatedMinorRecord) - .isPresent() - .get() - .extracting(r -> r.getValueString("firstName")) - .isEqualTo("Tim" + TestUtils.CheckAge.SUFFIX_FOR_MINORS); + ///////////////////////////////////////// + // make sure both persons were updated // + ///////////////////////////////////////// + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)) + .allMatch(r -> r.getValueString("firstName").endsWith(TestUtils.CheckAge.SUFFIX_FOR_MINORS)); ///////////////////////////////////////////////////////////////////////////////////////// // run automations again - make sure that there haven't been any updates triggered yet // @@ -179,7 +208,7 @@ class PollingAutomationRunnerTest /******************************************************************************* - ** Test a cycle that does an insert, some automations, then and an update, and more automations. + ** Test running a process for automation, instead of a code ref. *******************************************************************************/ @Test void testRunningProcess() throws QException @@ -187,9 +216,9 @@ class PollingAutomationRunnerTest QInstance qInstance = TestUtils.defineInstance(); PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, TestUtils.POLLING_AUTOMATION, null); - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // insert 2 person records, one who should be both updated by the insert action, and should be logged by logger-on-update automation // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////// + // insert 2 person records, 1 to trigger the "increaseAge" action // + //////////////////////////////////////////////////////////////////// InsertInput insertInput = new InsertInput(qInstance); insertInput.setSession(new QSession()); insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); @@ -214,6 +243,62 @@ class PollingAutomationRunnerTest } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRunningEtlWithFrontendProcess() throws QException + { + QInstance instance = TestUtils.defineInstance(); + + //////////////////////////////////////////////////////// + // define the process - an ELT from Shapes to Persons // + //////////////////////////////////////////////////////// + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( + TestUtils.TABLE_NAME_SHAPE, + TestUtils.TABLE_NAME_PERSON, + ExtractViaQueryStep.class, + StreamedETLWithFrontendProcessTest.TestTransformShapeToPersonStep.class, + LoadViaInsertStep.class); + process.setName("shapeToPersonETLProcess"); + process.setTableName(TestUtils.TABLE_NAME_SHAPE); + instance.addProcess(process); + + /////////////////////////////////////////////////////// + // switch the person table to use the memory backend // + /////////////////////////////////////////////////////// + instance.getTable(TestUtils.TABLE_NAME_PERSON).setBackendName(TestUtils.MEMORY_BACKEND_NAME); + + /////////////////////////////////////////////////////////////////////// + // add a post-insert process to the shape table, to run this ELT job // + /////////////////////////////////////////////////////////////////////// + instance.getTable(TestUtils.TABLE_NAME_SHAPE) + .withField(new QFieldMetaData("automationStatus", QFieldType.INTEGER)) + .setAutomationDetails(new QTableAutomationDetails() + .withProviderName(TestUtils.POLLING_AUTOMATION) + .withStatusTracking(new AutomationStatusTracking().withType(AutomationStatusTrackingType.FIELD_IN_TABLE).withFieldName("automationStatus")) + .withAction(new TableAutomationAction() + .withName("shapeToPerson") + .withTriggerEvent(TriggerEvent.POST_INSERT) + .withProcessName("shapeToPersonETLProcess") + ) + ); + + TestUtils.insertDefaultShapes(instance); + + PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(instance, TestUtils.POLLING_AUTOMATION, null); + pollingAutomationRunner.run(); + + List postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_PERSON); + assertThat(postList) + .as("Should have inserted Circle").anyMatch(qr -> qr.getValue("lastName").equals("Circle")) + .as("Should have inserted Triangle").anyMatch(qr -> qr.getValue("lastName").equals("Triangle")) + .as("Should have inserted Square").anyMatch(qr -> qr.getValue("lastName").equals("Square")); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/PersonsByCreateDateBarChart.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/PersonsByCreateDateBarChart.java index 06997d0a..8ba67623 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/PersonsByCreateDateBarChart.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/PersonsByCreateDateBarChart.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.dashboard; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoaderTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoaderTest.java index c571e361..fbc6412f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoaderTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoaderTest.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.actions.dashboard; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java index efa5f6c6..692aac89 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java @@ -372,9 +372,9 @@ public class RunProcessTest )) ); - ///////////////////////////////////////////////////////////////////////////// - // make sure that if we run by default, we get stop on both frontend steps // - ///////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + // make sure that if we run by default, we get stopped on both frontend steps // + //////////////////////////////////////////////////////////////////////////////// RunProcessInput request = new RunProcessInput(qInstance); request.setSession(TestUtils.getMockSession()); request.setProcessName(processName); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index b2b3d95e..9a4e2f9e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -721,9 +721,8 @@ class QInstanceValidatorTest assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().getStatusTracking().setFieldName(""), "missing its fieldName"); - ////////////////////////////////////////////////// - // todo - make sure it's a field in the table?? // - ////////////////////////////////////////////////// + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().getStatusTracking().setFieldName("notARealField"), + "not a defined field"); } @@ -791,14 +790,6 @@ class QInstanceValidatorTest }, "unrecognized processName"); - assertValidationSuccess((qInstance) -> - { - qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); - TableAutomationAction action = getAction0(qInstance); - action.setCodeReference(null); - action.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE); - }); - assertValidationFailureReasons((qInstance) -> { TableAutomationAction action = getAction0(qInstance); @@ -826,9 +817,9 @@ class QInstanceValidatorTest assertValidationFailureReasons((qInstance) -> { - qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); TableAutomationAction action = getAction0(qInstance); - action.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE); + action.setCodeReference(new QCodeReference(TestUtils.CheckAge.class)); + action.setProcessName(TestUtils.PROCESS_NAME_INCREASE_BIRTHDATE); }, "has both"); } @@ -857,14 +848,6 @@ class QInstanceValidatorTest ); }, "unrecognized field"); - - assertValidationSuccess((qInstance) -> - { - TableAutomationAction action = getAction0(qInstance); - action.setFilter(new QQueryFilter() - .withCriteria(new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(1701))) - ); - }); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java index 7650099f..cce85d3d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; @@ -36,7 +57,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* ** Unit test for StreamedETLWithFrontendProcess *******************************************************************************/ -class StreamedETLWithFrontendProcessTest +public class StreamedETLWithFrontendProcessTest { /******************************************************************************* @@ -64,7 +85,7 @@ class StreamedETLWithFrontendProcessTest //////////////////////////////////////////////////////// QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( TestUtils.TABLE_NAME_SHAPE, - TestUtils.TABLE_NAME_PERSON, + TestUtils.TABLE_NAME_PERSON_MEMORY, ExtractViaQueryStep.class, TestTransformShapeToPersonStep.class, LoadViaInsertStep.class); @@ -72,11 +93,6 @@ class StreamedETLWithFrontendProcessTest process.setTableName(TestUtils.TABLE_NAME_SHAPE); instance.addProcess(process); - /////////////////////////////////////////////////////// - // switch the person table to use the memory backend // - /////////////////////////////////////////////////////// - instance.getTable(TestUtils.TABLE_NAME_PERSON).setBackendName(TestUtils.MEMORY_BACKEND_NAME); - TestUtils.insertDefaultShapes(instance); ///////////////////// @@ -84,7 +100,7 @@ class StreamedETLWithFrontendProcessTest ///////////////////// runProcess(instance, process); - List postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_PERSON); + List postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_PERSON_MEMORY); assertThat(postList) .as("Should have inserted Circle").anyMatch(qr -> qr.getValue("lastName").equals("Circle")) .as("Should have inserted Triangle").anyMatch(qr -> qr.getValue("lastName").equals("Triangle")) @@ -277,7 +293,7 @@ class StreamedETLWithFrontendProcessTest /******************************************************************************* ** *******************************************************************************/ - private RunProcessOutput runProcess(QInstance instance, QProcessMetaData process) throws QException + public RunProcessOutput runProcess(QInstance instance, QProcessMetaData process) throws QException { return (runProcess(instance, process, new HashMap<>(), new Callback())); } @@ -338,6 +354,14 @@ class StreamedETLWithFrontendProcessTest getOutputRecordPage().add(newQRecord); } } + + + + @Override + public ArrayList getProcessSummary(boolean isForResultScreen) + { + return null; + } } @@ -420,6 +444,14 @@ class StreamedETLWithFrontendProcessTest getOutputRecordPage().add(updatedQRecord); } } + + + + @Override + public ArrayList getProcessSummary(boolean isForResultScreen) + { + return null; + } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/FilesystemTableCustomizers.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/FilesystemTableCustomizers.java index 06157ceb..ea475a93 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/FilesystemTableCustomizers.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/FilesystemTableCustomizers.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.module.filesystem.base.actions; diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java index 5afbf0d6..18277f6f 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.qqq.backend.javalin; diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java index 490ac39f..decad446 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.sampleapp.dashboard.widgets; diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java index a8874be4..d0807971 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.sampleapp.processes.clonepeople; diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleJavalinServerTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleJavalinServerTest.java index 0a861b18..0c6310d7 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleJavalinServerTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleJavalinServerTest.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.sampleapp; diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChartTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChartTest.java index b74f98ba..9cfb1d47 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChartTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChartTest.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.sampleapp.dashboard.widgets; diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java index e27131a6..1ea36c1c 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java @@ -1,3 +1,24 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package com.kingsrook.sampleapp.processes.clonepeople; From 12925127b2bdc59728ee3cc5f5e4480ed5e773c4 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 6 Sep 2022 09:29:24 -0500 Subject: [PATCH 23/28] Feedback from code reviews --- .../metadata/processes/QProcessMetaData.java | 20 ++++++++++++++++++- .../ExtractViaQueryStep.java | 1 - .../StreamedETLWithFrontendProcess.java | 2 +- .../StreamedETLWithFrontendProcessTest.java | 3 ++- .../sampleapp/SampleMetaDataProvider.java | 2 -- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index 45247881..182e7496 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -198,12 +198,30 @@ public class QProcessMetaData implements QAppChildMetaData ** *******************************************************************************/ public QProcessMetaData addStep(QStepMetaData step) + { + int index = 0; + if(this.stepList != null) + { + index = this.stepList.size(); + } + addStep(index, step); + + return (this); + } + + + + /******************************************************************************* + ** add a step to the stepList (at the specified index) and the step map + ** + *******************************************************************************/ + public QProcessMetaData addStep(int index, QStepMetaData step) { if(this.stepList == null) { this.stepList = new ArrayList<>(); } - this.stepList.add(step); + this.stepList.add(index, step); if(this.steps == null) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java index aa94712a..888857d8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java @@ -33,7 +33,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 7ae50735..5d872b7c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -127,7 +127,7 @@ public class StreamedETLWithFrontendProcess .withInputData(new QFunctionInputMetaData() .withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE))) .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE))) - .withField(new QFieldMetaData(FIELD_SUPPORTS_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, false))) + .withField(new QFieldMetaData(FIELD_SUPPORTS_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, true))) .withField(new QFieldMetaData(FIELD_DEFAULT_QUERY_FILTER, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DEFAULT_QUERY_FILTER))) .withField(new QFieldMetaData(FIELD_EXTRACT_CODE, QFieldType.STRING).withDefaultValue(new QCodeReference(extractStepClass))) .withField(new QFieldMetaData(FIELD_TRANSFORM_CODE, QFieldType.STRING).withDefaultValue(new QCodeReference(transformStepClass))) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java index cce85d3d..5508520a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -246,7 +247,7 @@ public class StreamedETLWithFrontendProcessTest /////////////////////////////////////////////////////////////////////////// // run the process - breaking on the first instance of the Review screen // /////////////////////////////////////////////////////////////////////////// - RunProcessOutput runProcessOutput = runProcess(instance, process, Map.of(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true), new Callback(), RunProcessInput.FrontendStepBehavior.BREAK); + RunProcessOutput runProcessOutput = runProcess(instance, process, Collections.emptyMap(), new Callback(), RunProcessInput.FrontendStepBehavior.BREAK); assertThat(runProcessOutput.getProcessState().getNextStepName()).hasValue("review"); //////////////////////////////////////////////////////// diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index d90bec2e..f9175e14 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -72,7 +72,6 @@ import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.Filesyst import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.kingsrook.sampleapp.dashboard.widgets.PersonsByCreateDateBarChart; import com.kingsrook.sampleapp.processes.clonepeople.ClonePeopleTransformStep; -import io.github.cdimascio.dotenv.Dotenv; /******************************************************************************* @@ -445,7 +444,6 @@ public class SampleMetaDataProvider Map values = new HashMap<>(); values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, TABLE_NAME_PERSON); values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, TABLE_NAME_PERSON); - values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true); values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, "This is a preview of what the clones will look like."); QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( From 31e6bf4d4988893114320f34c00736336d0ec343 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 6 Sep 2022 11:41:25 -0500 Subject: [PATCH 24/28] PRDONE-94: updates from code review feedback added .env test --- .../actions/dashboard/AbstractWidgetRenderer.java | 1 + .../dashboard/QuickSightChartRenderer.java | 2 ++ .../core/actions/dashboard/WidgetDataLoader.java | 1 + .../core/model/dashboard/widgets/BarChart.java | 1 + .../core/model/dashboard/widgets/QWidget.java | 1 + .../model/dashboard/widgets/QuickSightChart.java | 2 ++ .../model/metadata/dashboard/QWidgetMetaData.java | 1 + .../dashboard/QWidgetMetaDataInterface.java | 1 + .../dashboard/QuickSightChartMetaData.java | 1 + .../QMetaDataVariableInterpreterTest.java | 15 +++++++++++++++ 10 files changed, 26 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java index 93370857..ba65c767 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; /******************************************************************************* + ** Base class for rendering qqq dashboard widgets ** *******************************************************************************/ public abstract class AbstractWidgetRenderer diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java index 5ce05407..6d8a50db 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java @@ -39,10 +39,12 @@ import software.amazon.awssdk.services.quicksight.model.RegisteredUserEmbeddingE /******************************************************************************* + ** Widget implementation for amazon QuickSight charts ** *******************************************************************************/ public class QuickSightChartRenderer extends AbstractWidgetRenderer { + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java index 83d2676c..293df38e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java @@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; /******************************************************************************* + ** Class for loading widget implementation code and rendering of widgets ** *******************************************************************************/ public class WidgetDataLoader diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java index 3fd0b73f..5c5ccf4b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java @@ -26,6 +26,7 @@ import java.util.List; /******************************************************************************* + ** Model containing datastructure expected by frontend material dashboard bar chart widget ** *******************************************************************************/ public class BarChart implements QWidget diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java index 81f1d400..d62c35df 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets; /******************************************************************************* + ** Interface for frontend material dashboard widget's datastructures ** *******************************************************************************/ public interface QWidget diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java index e5561cac..de738d6f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java @@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets; /******************************************************************************* + ** Model containing datastructure expected by frontend material dashboard + ** AWS quick sight widget ** *******************************************************************************/ public class QuickSightChart implements QWidget diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java index 3bf01462..7234eb90 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java @@ -26,6 +26,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; /******************************************************************************* + ** Base metadata for frontend material dashboard widgets ** *******************************************************************************/ public class QWidgetMetaData implements QWidgetMetaDataInterface diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java index 162fc56a..584e61b7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java @@ -26,6 +26,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; /******************************************************************************* + ** Interface for qqq widget meta data ** *******************************************************************************/ public interface QWidgetMetaDataInterface diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java index c83ff288..6d1b82f6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java @@ -26,6 +26,7 @@ import java.util.Collection; /******************************************************************************* + ** AWS Quicksite specific meta data for frontend material dashboard widget ** *******************************************************************************/ public class QuickSightChartMetaData extends QWidgetMetaData implements QWidgetMetaDataInterface diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java index d40a82d9..1e339199 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java @@ -113,6 +113,21 @@ class QMetaDataVariableInterpreterTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDotEnvFile() + { + QMetaDataVariableInterpreter secretReader = new QMetaDataVariableInterpreter(); + String key = "CUSTOM_PROPERTY"; + String value = "ABCD-9876"; + assertNull(secretReader.interpret("${env.NOT-" + key + "}")); + assertEquals(value, secretReader.interpret("${env." + key + "}")); + } + + + /******************************************************************************* ** *******************************************************************************/ From 25c9376ce474ba4cbaab65b48aa4282bc3f73562 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 6 Sep 2022 15:15:04 -0500 Subject: [PATCH 25/28] QQQ-37 update streamed-etl steps to not have to use different record-list --- .../processes/RunBackendStepInput.java | 24 ++++++ .../AbstractLoadStep.java | 58 ++------------ .../AbstractTransformStep.java | 58 ++------------ .../LoadViaInsertStep.java | 4 +- .../LoadViaUpdateStep.java | 4 +- .../StreamedBackendStepInput.java | 77 +++++++++++++++++++ .../StreamedBackendStepOutput.java | 74 ++++++++++++++++++ .../StreamedETLExecuteStep.java | 21 +++-- .../StreamedETLPreviewStep.java | 11 ++- .../StreamedETLValidateStep.java | 13 +++- .../StreamedETLWithFrontendProcessTest.java | 12 +-- .../clonepeople/ClonePeopleTransformStep.java | 4 +- .../ClonePeopleTransformStepTest.java | 2 +- 13 files changed, 229 insertions(+), 133 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepInput.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepOutput.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index cab801b9..69cf5787 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -51,6 +51,10 @@ public class RunBackendStepInput extends AbstractActionInput private QProcessCallback callback; private AsyncJobCallback asyncJobCallback; + //////////////////////////////////////////////////////////////////////////// + // note - new fields should generally be added in method: cloneFieldsInto // + //////////////////////////////////////////////////////////////////////////// + /******************************************************************************* @@ -85,6 +89,25 @@ public class RunBackendStepInput extends AbstractActionInput + /******************************************************************************* + ** Kinda like a reverse copy-constructor -- for a subclass that wants all the + ** field values from this object. Keep this in sync with the fields in this class! + ** + ** Of note - the processState does NOT get cloned - because... well, in our first + ** use-case (a subclass that doesn't WANT the same/full state), that's what we needed. + *******************************************************************************/ + public void cloneFieldsInto(RunBackendStepInput target) + { + target.setStepName(getStepName()); + target.setSession(getSession()); + target.setTableName(getTableName()); + target.setProcessName(getProcessName()); + target.setAsyncJobCallback(getAsyncJobCallback()); + target.setValues(getValues()); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -429,4 +452,5 @@ public class RunBackendStepInput extends AbstractActionInput } return (asyncJobCallback); } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java index 55b82768..58dcbc3a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java @@ -22,31 +22,27 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; -import java.util.ArrayList; -import java.util.List; import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; /******************************************************************************* ** Base class for the Load (aka, store) logic of Streamed ETL processes. ** - ** Records are to be read out of the inputRecordPage field, and after storing, - ** should be written to the outputRecordPage. That is to say, DO NOT use the - ** recordList in the step input/output objects. + ** Records are to be read out of the input object's Records field, and after storing, + ** should be written to the output object's Records, noting that when running + ** as a streamed-ETL process, those input & output objects will be instances of + ** the StreamedBackendStep{Input,Output} classes, that will be associated with + ** a page of records flowing thorugh a pipe. ** ** Also - use the transaction member variable!!! *******************************************************************************/ public abstract class AbstractLoadStep implements BackendStep { - private List inputRecordPage = new ArrayList<>(); - private List outputRecordPage = new ArrayList<>(); - private Optional transaction = Optional.empty(); @@ -87,50 +83,6 @@ public abstract class AbstractLoadStep implements BackendStep - /******************************************************************************* - ** Getter for recordPage - ** - *******************************************************************************/ - public List getInputRecordPage() - { - return inputRecordPage; - } - - - - /******************************************************************************* - ** Setter for recordPage - ** - *******************************************************************************/ - public void setInputRecordPage(List inputRecordPage) - { - this.inputRecordPage = inputRecordPage; - } - - - - /******************************************************************************* - ** Getter for outputRecordPage - ** - *******************************************************************************/ - public List getOutputRecordPage() - { - return outputRecordPage; - } - - - - /******************************************************************************* - ** Setter for outputRecordPage - ** - *******************************************************************************/ - public void setOutputRecordPage(List outputRecordPage) - { - this.outputRecordPage = outputRecordPage; - } - - - /******************************************************************************* ** Setter for transaction ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java index ffe4e7ef..200f895a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java @@ -22,27 +22,24 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; -import java.util.ArrayList; -import java.util.List; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; /******************************************************************************* ** Base class for the Transform logic of Streamed ETL processes. ** - ** Records are to be read out of the inputRecordPage field, and after transformation, - ** should be written to the outputRecordPage. That is to say, DO NOT use the - ** recordList in the step input/output objects. + ** Records are to be read out of the input object's Records field, and after storing, + ** should be written to the output object's Records, noting that when running + ** as a streamed-ETL process, those input & output objects will be instances of + ** the StreamedBackendStep{Input,Output} classes, that will be associated with + ** a page of records flowing through a pipe. + ** *******************************************************************************/ public abstract class AbstractTransformStep implements BackendStep, ProcessSummaryProviderInterface { - private List inputRecordPage = new ArrayList<>(); - private List outputRecordPage = new ArrayList<>(); - /******************************************************************************* @@ -69,47 +66,4 @@ public abstract class AbstractTransformStep implements BackendStep, ProcessSumma //////////////////////// } - - - /******************************************************************************* - ** Getter for recordPage - ** - *******************************************************************************/ - public List getInputRecordPage() - { - return inputRecordPage; - } - - - - /******************************************************************************* - ** Setter for recordPage - ** - *******************************************************************************/ - public void setInputRecordPage(List inputRecordPage) - { - this.inputRecordPage = inputRecordPage; - } - - - - /******************************************************************************* - ** Getter for outputRecordPage - ** - *******************************************************************************/ - public List getOutputRecordPage() - { - return outputRecordPage; - } - - - - /******************************************************************************* - ** Setter for outputRecordPage - ** - *******************************************************************************/ - public void setOutputRecordPage(List outputRecordPage) - { - this.outputRecordPage = outputRecordPage; - } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java index fe0f1a29..4903a710 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java @@ -52,10 +52,10 @@ public class LoadViaInsertStep extends AbstractLoadStep InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance()); insertInput.setSession(runBackendStepInput.getSession()); insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE)); - insertInput.setRecords(getInputRecordPage()); + insertInput.setRecords(runBackendStepInput.getRecords()); getTransaction().ifPresent(insertInput::setTransaction); InsertOutput insertOutput = new InsertAction().execute(insertInput); - getOutputRecordPage().addAll(insertOutput.getRecords()); + runBackendStepOutput.getRecords().addAll(insertOutput.getRecords()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java index 69eaffd1..39572ee1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java @@ -54,10 +54,10 @@ public class LoadViaUpdateStep extends AbstractLoadStep UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance()); updateInput.setSession(runBackendStepInput.getSession()); updateInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE)); - updateInput.setRecords(getInputRecordPage()); + updateInput.setRecords(runBackendStepInput.getRecords()); getTransaction().ifPresent(updateInput::setTransaction); UpdateOutput updateOutput = new UpdateAction().execute(updateInput); - getOutputRecordPage().addAll(updateOutput.getRecords()); + runBackendStepOutput.getRecords().addAll(updateOutput.getRecords()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepInput.java new file mode 100644 index 00000000..14b8f10e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepInput.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Subclass of RunBackendStepInput, meant for use in the pseudo-steps used by + ** the Streamed-ETL-with-frontend processes - where the Record list is not the + ** full process's record list - rather - is just a page at a time -- so this class + ** overrides the getRecords and setRecords method, to just work with that page. + ** + ** Note - of importance over time may be the RunBackendStepInput::cloneFieldsInto + ** method - e.g., if new fields are added to that class! + *******************************************************************************/ +public class StreamedBackendStepInput extends RunBackendStepInput +{ + private List inputRecords; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public StreamedBackendStepInput(RunBackendStepInput runBackendStepInput, List inputRecords) + { + super(runBackendStepInput.getInstance()); + runBackendStepInput.cloneFieldsInto(this); + this.inputRecords = inputRecords; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setRecords(List records) + { + this.inputRecords = records; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List getRecords() + { + return (inputRecords); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepOutput.java new file mode 100644 index 00000000..f750e67e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedBackendStepOutput.java @@ -0,0 +1,74 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** Subclass of RunBackendStepOutput, meant for use in the pseudo-steps used by + ** the Streamed-ETL-with-frontend processes - where the Record list is not the + ** full process's record list - rather - is just a page at a time -- so this class + ** overrides the getRecords and setRecords method, to just work with that page. + *******************************************************************************/ +public class StreamedBackendStepOutput extends RunBackendStepOutput +{ + private List outputRecords = new ArrayList<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public StreamedBackendStepOutput(RunBackendStepOutput runBackendStepOutput) + { + super(); + setValues(runBackendStepOutput.getValues()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void setRecords(List records) + { + this.outputRecords = records; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public List getRecords() + { + return (outputRecords); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java index 2c5f413f..4f33651f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java @@ -144,27 +144,32 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe /////////////////////////////////// List qRecords = recordPipe.consumeAvailableRecords(); + /////////////////////////////////////////////////////////////////////// + // make streamed input & output objects from the run input & outputs // + /////////////////////////////////////////////////////////////////////// + StreamedBackendStepInput streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, qRecords); + StreamedBackendStepOutput streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput); + ///////////////////////////////////////////////////// // pass the records through the transform function // ///////////////////////////////////////////////////// - transformStep.setInputRecordPage(qRecords); - transformStep.setOutputRecordPage(new ArrayList<>()); - transformStep.run(runBackendStepInput, runBackendStepOutput); + transformStep.run(streamedBackendStepInput, streamedBackendStepOutput); //////////////////////////////////////////////// // pass the records through the load function // //////////////////////////////////////////////// - loadStep.setInputRecordPage(transformStep.getOutputRecordPage()); - loadStep.setOutputRecordPage(new ArrayList<>()); - loadStep.run(runBackendStepInput, runBackendStepOutput); + streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, streamedBackendStepOutput.getRecords()); + streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput); + + loadStep.run(streamedBackendStepInput, streamedBackendStepOutput); /////////////////////////////////////////////////////// // copy a small number of records to the output list // /////////////////////////////////////////////////////// int i = 0; - while(loadedRecordList.size() < PROCESS_OUTPUT_RECORD_LIST_LIMIT && i < loadStep.getOutputRecordPage().size()) + while(loadedRecordList.size() < PROCESS_OUTPUT_RECORD_LIST_LIMIT && i < streamedBackendStepOutput.getRecords().size()) { - loadedRecordList.add(loadStep.getOutputRecordPage().get(i++)); + loadedRecordList.add(streamedBackendStepOutput.getRecords().get(i++)); } currentRowCount += qRecords.size(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java index 74367ec4..7dd6c185 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -151,16 +151,21 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe /////////////////////////////////// List qRecords = recordPipe.consumeAvailableRecords(); + /////////////////////////////////////////////////////////////////////// + // make streamed input & output objects from the run input & outputs // + /////////////////////////////////////////////////////////////////////// + StreamedBackendStepInput streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, qRecords); + StreamedBackendStepOutput streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput); + ///////////////////////////////////////////////////// // pass the records through the transform function // ///////////////////////////////////////////////////// - transformStep.setInputRecordPage(qRecords); - transformStep.run(runBackendStepInput, runBackendStepOutput); + transformStep.run(streamedBackendStepInput, streamedBackendStepOutput); //////////////////////////////////////////////////// // add the transformed records to the output list // //////////////////////////////////////////////////// - transformedRecordList.addAll(transformStep.getOutputRecordPage()); + transformedRecordList.addAll(streamedBackendStepOutput.getRecords()); return (qRecords.size()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java index adcd570b..1057c84c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java @@ -128,19 +128,24 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back /////////////////////////////////// List qRecords = recordPipe.consumeAvailableRecords(); + /////////////////////////////////////////////////////////////////////// + // make streamed input & output objects from the run input & outputs // + /////////////////////////////////////////////////////////////////////// + StreamedBackendStepInput streamedBackendStepInput = new StreamedBackendStepInput(runBackendStepInput, qRecords); + StreamedBackendStepOutput streamedBackendStepOutput = new StreamedBackendStepOutput(runBackendStepOutput); + ///////////////////////////////////////////////////// // pass the records through the transform function // ///////////////////////////////////////////////////// - transformStep.setInputRecordPage(qRecords); - transformStep.run(runBackendStepInput, runBackendStepOutput); + transformStep.run(streamedBackendStepInput, streamedBackendStepOutput); /////////////////////////////////////////////////////// // copy a small number of records to the output list // /////////////////////////////////////////////////////// int i = 0; - while(previewRecordList.size() < PROCESS_OUTPUT_RECORD_LIST_LIMIT && i < transformStep.getOutputRecordPage().size()) + while(previewRecordList.size() < PROCESS_OUTPUT_RECORD_LIST_LIMIT && i < streamedBackendStepOutput.getRecords().size()) { - previewRecordList.add(transformStep.getOutputRecordPage().get(i++)); + previewRecordList.add(streamedBackendStepOutput.getRecords().get(i++)); } currentRowCount += qRecords.size(); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java index 5508520a..62cf8540 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java @@ -347,12 +347,12 @@ public class StreamedETLWithFrontendProcessTest @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - for(QRecord qRecord : getInputRecordPage()) + for(QRecord qRecord : runBackendStepInput.getRecords()) { QRecord newQRecord = new QRecord(); newQRecord.setValue("firstName", "Johnny"); newQRecord.setValue("lastName", qRecord.getValueString("name")); - getOutputRecordPage().add(newQRecord); + runBackendStepOutput.getRecords().add(newQRecord); } } @@ -403,7 +403,7 @@ public class StreamedETLWithFrontendProcessTest @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - for(QRecord qRecord : getInputRecordPage()) + for(QRecord qRecord : runBackendStepInput.getRecords()) { if(qRecord.getValueString("name").equals("Circle")) { @@ -414,7 +414,7 @@ public class StreamedETLWithFrontendProcessTest QRecord newQRecord = new QRecord(); newQRecord.setValue("firstName", "Johnny"); newQRecord.setValue("lastName", qRecord.getValueString("name")); - getOutputRecordPage().add(newQRecord); + runBackendStepOutput.getRecords().add(newQRecord); okSummary.incrementCountAndAddPrimaryKey(qRecord.getValue("id")); } @@ -437,12 +437,12 @@ public class StreamedETLWithFrontendProcessTest @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - for(QRecord qRecord : getInputRecordPage()) + for(QRecord qRecord : runBackendStepInput.getRecords()) { QRecord updatedQRecord = new QRecord(); updatedQRecord.setValue("id", qRecord.getValue("id")); updatedQRecord.setValue("name", "Transformed:" + qRecord.getValueString("name")); - getOutputRecordPage().add(updatedQRecord); + runBackendStepOutput.getRecords().add(updatedQRecord); } } diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java index d0807971..f3833670 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java @@ -76,7 +76,7 @@ public class ClonePeopleTransformStep extends AbstractTransformStep implements P @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - for(QRecord inputPerson : getInputRecordPage()) + for(QRecord inputPerson : runBackendStepInput.getRecords()) { Serializable id = inputPerson.getValue("id"); if("Garret".equals(inputPerson.getValueString("firstName"))) @@ -92,7 +92,7 @@ public class ClonePeopleTransformStep extends AbstractTransformStep implements P QRecord outputPerson = new QRecord(inputPerson); outputPerson.setValue("id", null); outputPerson.setValue("firstName", "Clone of: " + inputPerson.getValueString("firstName")); - getOutputRecordPage().add(outputPerson); + runBackendStepOutput.getRecords().add(outputPerson); if(inputPerson.getValueString("firstName").matches("Clone of.*")) { diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java index 1ea36c1c..a1c5758f 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java @@ -91,7 +91,7 @@ class ClonePeopleTransformStepTest RunBackendStepOutput output = new RunBackendStepOutput(); ClonePeopleTransformStep clonePeopleTransformStep = new ClonePeopleTransformStep(); - clonePeopleTransformStep.setInputRecordPage(queryOutput.getRecords()); + input.setRecords(queryOutput.getRecords()); clonePeopleTransformStep.run(input, output); ArrayList processSummary = clonePeopleTransformStep.getProcessSummary(true); From 1c75df3a0953b7d1778cf1c27e652a332341b0a1 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 6 Sep 2022 19:00:12 -0500 Subject: [PATCH 26/28] added initial version of branding as metadata --- .../core/actions/metadata/MetaDataAction.java | 8 + .../actions/metadata/MetaDataOutput.java | 26 ++- .../model/dashboard/widgets/BarChart.java | 2 +- .../core/model/dashboard/widgets/QWidget.java | 2 +- .../dashboard/widgets/QuickSightChart.java | 4 +- .../core/model/metadata/QInstance.java | 25 +++ .../metadata/branding/QBrandingMetaData.java | 148 ++++++++++++++++++ .../metadata/dashboard/QWidgetMetaData.java | 2 +- .../dashboard/QuickSightChartMetaData.java | 2 +- .../sampleapp/SampleMetaDataProvider.java | 37 +++-- 10 files changed, 237 insertions(+), 19 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java index ce21b93f..97d460ab 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java @@ -109,6 +109,14 @@ public class MetaDataAction } metaDataOutput.setAppTree(appTree); + //////////////////////////////////// + // add branding metadata if found // + //////////////////////////////////// + if(metaDataInput.getInstance().getBranding() != null) + { + metaDataOutput.setBranding(metaDataInput.getInstance().getBranding()); + } + // todo post-customization - can do whatever w/ the result if you want? return metaDataOutput; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java index 759ba708..afb3d709 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataOutput.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.metadata; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; +import com.kingsrook.qqq.backend.core.model.metadata.branding.QBrandingMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendProcessMetaData; @@ -43,6 +44,8 @@ public class MetaDataOutput extends AbstractActionOutput private List appTree; + private QBrandingMetaData branding; + /******************************************************************************* @@ -89,7 +92,6 @@ public class MetaDataOutput extends AbstractActionOutput - /******************************************************************************* ** Getter for appTree ** @@ -131,4 +133,26 @@ public class MetaDataOutput extends AbstractActionOutput { this.apps = apps; } + + + + /******************************************************************************* + ** Getter for branding + ** + *******************************************************************************/ + public QBrandingMetaData getBranding() + { + return branding; + } + + + + /******************************************************************************* + ** Setter for branding + ** + *******************************************************************************/ + public void setBranding(QBrandingMetaData branding) + { + this.branding = branding; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java index 5c5ccf4b..27f6689b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java @@ -26,7 +26,7 @@ import java.util.List; /******************************************************************************* - ** Model containing datastructure expected by frontend material dashboard bar chart widget + ** Model containing datastructure expected by frontend bar chart widget ** *******************************************************************************/ public class BarChart implements QWidget diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java index d62c35df..8eb0b721 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java @@ -23,7 +23,7 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets; /******************************************************************************* - ** Interface for frontend material dashboard widget's datastructures + ** Interface for frontend widget's datastructures ** *******************************************************************************/ public interface QWidget diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java index de738d6f..5aec5320 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java @@ -23,8 +23,8 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets; /******************************************************************************* - ** Model containing datastructure expected by frontend material dashboard - ** AWS quick sight widget + ** Model containing datastructure expected by frontend AWS quick sight widget + ** TODO: this might just be an IFrameChart widget in the future ** *******************************************************************************/ public class QuickSightChart implements QWidget diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index c221d013..1fd65d4c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -30,6 +30,7 @@ import java.util.Map; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.instances.QInstanceValidationKey; import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.branding.QBrandingMetaData; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; @@ -53,6 +54,7 @@ public class QInstance private Map backends = new HashMap<>(); private QAuthenticationMetaData authentication = null; + private QBrandingMetaData branding = null; private Map automationProviders = new HashMap<>(); //////////////////////////////////////////////////////////////////////////////////////////// @@ -508,6 +510,28 @@ public class QInstance + /******************************************************************************* + ** Getter for branding + ** + *******************************************************************************/ + public QBrandingMetaData getBranding() + { + return branding; + } + + + + /******************************************************************************* + ** Setter for branding + ** + *******************************************************************************/ + public void setBranding(QBrandingMetaData branding) + { + this.branding = branding; + } + + + /******************************************************************************* ** Getter for authentication ** @@ -551,6 +575,7 @@ public class QInstance } + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java new file mode 100644 index 00000000..c02b25fd --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java @@ -0,0 +1,148 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.branding; + + +/******************************************************************************* + ** Meta-Data to define branding in a QQQ instance. + ** + *******************************************************************************/ +public class QBrandingMetaData +{ + private String companyName; + private String logo; + private String icon; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String toString() + { + return ("QBrandingMetaData[" + companyName + "]"); + } + + + + /******************************************************************************* + ** Getter for companyName + ** + *******************************************************************************/ + public String getCompanyName() + { + return companyName; + } + + + + /******************************************************************************* + ** Setter for companyName + ** + *******************************************************************************/ + public void setCompanyName(String companyName) + { + this.companyName = companyName; + } + + + + /******************************************************************************* + ** Fluent setter for companyName + ** + *******************************************************************************/ + public QBrandingMetaData withCompanyName(String companyName) + { + this.companyName = companyName; + return this; + } + + + + /******************************************************************************* + ** Getter for logo + ** + *******************************************************************************/ + public String getLogo() + { + return logo; + } + + + + /******************************************************************************* + ** Setter for logo + ** + *******************************************************************************/ + public void setLogo(String logo) + { + this.logo = logo; + } + + + + /******************************************************************************* + ** Fluent setter for logo + ** + *******************************************************************************/ + public QBrandingMetaData withLogo(String logo) + { + this.logo = logo; + return this; + } + + + + /******************************************************************************* + ** Getter for icon + ** + *******************************************************************************/ + public String getIcon() + { + return icon; + } + + + + /******************************************************************************* + ** Setter for icon + ** + *******************************************************************************/ + public void setIcon(String icon) + { + this.icon = icon; + } + + + + /******************************************************************************* + ** Fluent setter for icon + ** + *******************************************************************************/ + public QBrandingMetaData withIcon(String icon) + { + this.icon = icon; + return this; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java index 7234eb90..95a2de76 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java @@ -26,7 +26,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; /******************************************************************************* - ** Base metadata for frontend material dashboard widgets + ** Base metadata for frontend dashboard widgets ** *******************************************************************************/ public class QWidgetMetaData implements QWidgetMetaDataInterface diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java index 6d1b82f6..55d7e31b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java @@ -26,7 +26,7 @@ import java.util.Collection; /******************************************************************************* - ** AWS Quicksite specific meta data for frontend material dashboard widget + ** AWS Quicksite specific meta data for frontend dashboard widget ** *******************************************************************************/ public class QuickSightChartMetaData extends QWidgetMetaData implements QWidgetMetaDataInterface diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index f9175e14..028262f8 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.branding.QBrandingMetaData; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage; @@ -128,7 +129,7 @@ public class SampleMetaDataProvider qInstance.addProcess(defineProcessSimpleThrow()); defineWidgets(qInstance); - + defineBranding(qInstance); defineApps(qInstance); return (qInstance); @@ -136,6 +137,18 @@ public class SampleMetaDataProvider + /******************************************************************************* + ** + *******************************************************************************/ + private static void defineBranding(QInstance qInstance) + { + qInstance.setBranding(new QBrandingMetaData() + .withLogo("/kr-logo.png") + .withIcon("/kr-icon.png")); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -146,10 +159,10 @@ public class SampleMetaDataProvider .withCodeReference(new QCodeReference(PersonsByCreateDateBarChart.class, null))); QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter(); - String accountId = interpreter.interpret("${env.QUICKSIGHT_ACCOUNT_ID}"); - String accessKey = interpreter.interpret("${env.QUICKSIGHT_ACCESS_KEY}"); - String secretKey = interpreter.interpret("${env.QUICKSIGHT_SECRET_KEY}"); - String userArn = interpreter.interpret("${env.QUICKSIGHT_USER_ARN}"); + String accountId = interpreter.interpret("${env.QUICKSIGHT_ACCOUNT_ID}"); + String accessKey = interpreter.interpret("${env.QUICKSIGHT_ACCESS_KEY}"); + String secretKey = interpreter.interpret("${env.QUICKSIGHT_SECRET_KEY}"); + String userArn = interpreter.interpret("${env.QUICKSIGHT_USER_ARN}"); QWidgetMetaDataInterface quickSightChartMetaData = new QuickSightChartMetaData() .withAccountId(accountId) @@ -219,13 +232,13 @@ public class SampleMetaDataProvider { if(USE_MYSQL) { - QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter(); - String vendor = interpreter.interpret("${env.RDBMS_VENDOR}"); - String hostname = interpreter.interpret("${env.RDBMS_HOSTNAME}"); - Integer port = Integer.valueOf(interpreter.interpret("${env.RDBMS_PORT}")); - String databaseName = interpreter.interpret("${env.RDBMS_DATABASE_NAME}"); - String username = interpreter.interpret("${env.RDBMS_USERNAME}"); - String password= interpreter.interpret("${env.RDBMS_PASSWORD}"); + QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter(); + String vendor = interpreter.interpret("${env.RDBMS_VENDOR}"); + String hostname = interpreter.interpret("${env.RDBMS_HOSTNAME}"); + Integer port = Integer.valueOf(interpreter.interpret("${env.RDBMS_PORT}")); + String databaseName = interpreter.interpret("${env.RDBMS_DATABASE_NAME}"); + String username = interpreter.interpret("${env.RDBMS_USERNAME}"); + String password = interpreter.interpret("${env.RDBMS_PASSWORD}"); return new RDBMSBackendMetaData() .withName(RDBMS_BACKEND_NAME) From b01758879c17f6fff58d9baf1b1654538be26321 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 7 Sep 2022 16:59:42 -0500 Subject: [PATCH 27/28] QQQ-37 Redo bulk processes in streamed-etl mode --- .../core/actions/values/QValueFormatter.java | 4 + .../core/adapters/CsvToQRecordAdapter.java | 288 +++++++++++++++++- .../core/instances/QInstanceEnricher.java | 201 ++++-------- .../actions/processes/ProcessSummaryLine.java | 25 +- .../actions/tables/delete/DeleteInput.java | 42 ++- .../bulk/delete/BulkDeleteStoreStep.java | 101 ------ .../bulk/delete/BulkDeleteTransformStep.java | 126 ++++++++ .../bulk/edit/BulkEditReceiveValuesStep.java | 70 ----- .../bulk/edit/BulkEditStoreRecordsStep.java | 90 ------ .../bulk/edit/BulkEditTransformStep.java | 215 +++++++++++++ .../bulk/edit/BulkEditUtils.java | 68 ----- ...tUtils.java => BulkInsertExtractStep.java} | 57 ++-- .../insert/BulkInsertReceiveFileStep.java | 49 --- .../bulk/insert/BulkInsertTransformStep.java | 119 ++++++++ .../ExtractViaQueryStep.java | 53 +++- .../LoadViaDeleteStep.java} | 49 ++- .../StreamedETLWithFrontendProcess.java | 1 + .../bulk/delete/BulkDeleteStoreStepTest.java | 74 ----- .../bulk/delete/BulkDeleteTest.java | 119 ++++++++ .../edit/BulkEditReceiveValuesStepTest.java | 78 ----- .../edit/BulkEditStoreRecordsStepTest.java | 85 ------ .../bulk/edit/BulkEditTest.java | 145 +++++++++ .../insert/BulkInsertReceiveFileStepTest.java | 114 ------- .../BulkInsertStoreRecordsStepTest.java | 80 ----- .../bulk/insert/BulkInsertTest.java | 121 ++++++++ .../qqq/backend/core/utils/TestUtils.java | 16 +- .../rdbms/actions/RDBMSDeleteAction.java | 76 +++-- 27 files changed, 1430 insertions(+), 1036 deletions(-) delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/{BulkInsertUtils.java => BulkInsertExtractStep.java} (70%) delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/{bulk/insert/BulkInsertStoreRecordsStep.java => etl/streamedwithfrontend/LoadViaDeleteStep.java} (51%) delete mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTest.java delete mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStepTest.java delete mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStepTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTest.java delete mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java delete mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index 5ac811a9..3af8b35d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -74,6 +74,10 @@ public class QValueFormatter { return formatValue(field, ValueUtils.getValueAsBigDecimal(value)); } + else if(e.getMessage().equals("f != java.lang.String")) + { + return formatValue(field, ValueUtils.getValueAsBigDecimal(value)); + } else if(e.getMessage().equals("d != java.math.BigDecimal")) { return formatValue(field, ValueUtils.getValueAsInteger(value)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java index 98f85478..77b1e85d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.function.Consumer; @@ -62,8 +63,7 @@ public class CsvToQRecordAdapter *******************************************************************************/ public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping mapping, Consumer recordCustomizer) { - this.recordPipe = recordPipe; - doBuildRecordsFromCsv(csv, table, mapping, recordCustomizer); + buildRecordsFromCsv(new InputWrapper().withRecordPipe(recordPipe).withCsv(csv).withTable(table).withMapping(mapping).withRecordCustomizer(recordCustomizer)); } @@ -75,8 +75,7 @@ public class CsvToQRecordAdapter *******************************************************************************/ public List buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping mapping) { - this.recordList = new ArrayList<>(); - doBuildRecordsFromCsv(csv, table, mapping, null); + buildRecordsFromCsv(new InputWrapper().withCsv(csv).withTable(table).withMapping(mapping)); return (recordList); } @@ -88,13 +87,29 @@ public class CsvToQRecordAdapter ** ** todo - meta-data validation, type handling *******************************************************************************/ - public void doBuildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping mapping, Consumer recordCustomizer) + public void buildRecordsFromCsv(InputWrapper inputWrapper) { + String csv = inputWrapper.getCsv(); + AbstractQFieldMapping mapping = inputWrapper.getMapping(); + Consumer recordCustomizer = inputWrapper.getRecordCustomizer(); + QTableMetaData table = inputWrapper.getTable(); + Integer limit = inputWrapper.getLimit(); + if(!StringUtils.hasContent(csv)) { throw (new IllegalArgumentException("Empty csv value was provided.")); } + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // if caller supplied a record pipe, use it -- but if it's null, then create a recordList to populate. // + // see addRecord method for usage. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + this.recordPipe = inputWrapper.getRecordPipe(); + if(this.recordPipe == null) + { + this.recordList = new ArrayList<>(); + } + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // once, from a DOS csv file (that had come from Excel), we had a "" character (FEFF, Byte-order marker) at the start of a // // CSV, which caused our first header to not match... So, let us strip away any FEFF or FFFE's at the start of CSV strings. // @@ -120,9 +135,12 @@ public class CsvToQRecordAdapter List headers = csvParser.getHeaderNames(); headers = makeHeadersUnique(headers); - List csvRecords = csvParser.getRecords(); - for(CSVRecord csvRecord : csvRecords) + Iterator csvIterator = csvParser.iterator(); + int recordCount = 0; + while(csvIterator.hasNext()) { + CSVRecord csvRecord = csvIterator.next(); + ////////////////////////////////////////////////////////////////// // put values from the CSV record into a map of header -> value // ////////////////////////////////////////////////////////////////// @@ -144,6 +162,12 @@ public class CsvToQRecordAdapter runRecordCustomizer(recordCustomizer, qRecord); addRecord(qRecord); + + recordCount++; + if(limit != null && recordCount > limit) + { + break; + } } } else if(AbstractQFieldMapping.SourceType.INDEX.equals(mapping.getSourceType())) @@ -155,9 +179,12 @@ public class CsvToQRecordAdapter CSVFormat.DEFAULT .withTrim()); - List csvRecords = csvParser.getRecords(); - for(CSVRecord csvRecord : csvRecords) + Iterator csvIterator = csvParser.iterator(); + int recordCount = 0; + while(csvIterator.hasNext()) { + CSVRecord csvRecord = csvIterator.next(); + ///////////////////////////////////////////////////////////////// // put values from the CSV record into a map of index -> value // ///////////////////////////////////////////////////////////////// @@ -180,6 +207,12 @@ public class CsvToQRecordAdapter runRecordCustomizer(recordCustomizer, qRecord); addRecord(qRecord); + + recordCount++; + if(limit != null && recordCount > limit) + { + break; + } } } else @@ -261,4 +294,241 @@ public class CsvToQRecordAdapter } } + + + /******************************************************************************* + ** Getter for recordList - note - only is valid if you don't supply a pipe in + ** the input. If you do supply a pipe, then you get an exception if you call here! + ** + *******************************************************************************/ + public List getRecordList() + { + if(recordPipe != null) + { + throw (new IllegalStateException("getRecordList called on a CSVToQRecordAdapter that ran with a recordPipe.")); + } + + return recordList; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class InputWrapper + { + private RecordPipe recordPipe; + private String csv; + private QTableMetaData table; + private AbstractQFieldMapping mapping; + private Consumer recordCustomizer; + private Integer limit; + + + + /******************************************************************************* + ** Getter for recordPipe + ** + *******************************************************************************/ + public RecordPipe getRecordPipe() + { + return recordPipe; + } + + + + /******************************************************************************* + ** Setter for recordPipe + ** + *******************************************************************************/ + public void setRecordPipe(RecordPipe recordPipe) + { + this.recordPipe = recordPipe; + } + + + + /******************************************************************************* + ** Fluent setter for recordPipe + ** + *******************************************************************************/ + public InputWrapper withRecordPipe(RecordPipe recordPipe) + { + this.recordPipe = recordPipe; + return (this); + } + + + + /******************************************************************************* + ** Getter for csv + ** + *******************************************************************************/ + public String getCsv() + { + return csv; + } + + + + /******************************************************************************* + ** Setter for csv + ** + *******************************************************************************/ + public void setCsv(String csv) + { + this.csv = csv; + } + + + + /******************************************************************************* + ** Fluent setter for csv + ** + *******************************************************************************/ + public InputWrapper withCsv(String csv) + { + this.csv = csv; + return (this); + } + + + + /******************************************************************************* + ** Getter for table + ** + *******************************************************************************/ + public QTableMetaData getTable() + { + return table; + } + + + + /******************************************************************************* + ** Setter for table + ** + *******************************************************************************/ + public void setTable(QTableMetaData table) + { + this.table = table; + } + + + + /******************************************************************************* + ** Fluent setter for table + ** + *******************************************************************************/ + public InputWrapper withTable(QTableMetaData table) + { + this.table = table; + return (this); + } + + + + /******************************************************************************* + ** Getter for mapping + ** + *******************************************************************************/ + public AbstractQFieldMapping getMapping() + { + return mapping; + } + + + + /******************************************************************************* + ** Setter for mapping + ** + *******************************************************************************/ + public void setMapping(AbstractQFieldMapping mapping) + { + this.mapping = mapping; + } + + + + /******************************************************************************* + ** Fluent setter for mapping + ** + *******************************************************************************/ + public InputWrapper withMapping(AbstractQFieldMapping mapping) + { + this.mapping = mapping; + return (this); + } + + + + /******************************************************************************* + ** Getter for recordCustomizer + ** + *******************************************************************************/ + public Consumer getRecordCustomizer() + { + return recordCustomizer; + } + + + + /******************************************************************************* + ** Setter for recordCustomizer + ** + *******************************************************************************/ + public void setRecordCustomizer(Consumer recordCustomizer) + { + this.recordCustomizer = recordCustomizer; + } + + + + /******************************************************************************* + ** Fluent setter for recordCustomizer + ** + *******************************************************************************/ + public InputWrapper withRecordCustomizer(Consumer recordCustomizer) + { + this.recordCustomizer = recordCustomizer; + return (this); + } + + + + /******************************************************************************* + ** Getter for limit + ** + *******************************************************************************/ + public Integer getLimit() + { + return limit; + } + + + + /******************************************************************************* + ** Setter for limit + ** + *******************************************************************************/ + public void setLimit(Integer limit) + { + this.limit = limit; + } + + + + /******************************************************************************* + ** Fluent setter for limit + ** + *******************************************************************************/ + public InputWrapper withLimit(Integer limit) + { + this.limit = limit; + return (this); + } + + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 765d26b8..d56f830a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -22,38 +22,39 @@ package com.kingsrook.qqq.backend.core.instances; +import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteStoreStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditReceiveValuesStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditStoreRecordsStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertStoreRecordsStep; -import com.kingsrook.qqq.backend.core.processes.implementations.general.LoadInitialRecordsStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaUpdateStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import org.apache.logging.log4j.LogManager; @@ -293,6 +294,20 @@ public class QInstanceEnricher *******************************************************************************/ private void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName) { + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); + + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( + BulkInsertExtractStep.class, + BulkInsertTransformStep.class, + LoadViaInsertStep.class, + values + ) + .withName(processName) + .withLabel(table.getLabel() + " Bulk Insert") + .withTableName(table.getName()) + .withIsHidden(true); + List editableFields = table.getFields().values().stream() .filter(QFieldMetaData::getIsEditable) .toList(); @@ -307,54 +322,13 @@ public class QInstanceEnricher .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withIsRequired(true)) .withComponent(new QFrontendComponentMetaData() .withType(QComponentType.HELP_TEXT) - .withValue("text", "Upload a CSV file with the following columns: " + fieldsForHelpText)) + .withValue("previewText", "file upload instructions") + .withValue("text", "Upload a CSV file with the following columns:\n" + fieldsForHelpText)) .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)); - QBackendStepMetaData receiveFileStep = new QBackendStepMetaData() - .withName("receiveFile") - .withCode(new QCodeReference(BulkInsertReceiveFileStep.class)) - .withOutputMetaData(new QFunctionOutputMetaData() - .withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER)))); - - QFrontendStepMetaData reviewScreen = new QFrontendStepMetaData() - .withName("review") - .withRecordListFields(editableFields) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below were parsed from your file, and will be inserted if you click Submit.")) - .withViewField(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER).withLabel("# of file rows")) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); - - QBackendStepMetaData storeStep = new QBackendStepMetaData() - .withName("storeRecords") - .withCode(new QCodeReference(BulkInsertStoreRecordsStep.class)) - .withOutputMetaData(new QFunctionOutputMetaData() - .withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER)))); - - QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData() - .withName("results") - .withRecordListFields(new ArrayList<>(table.getFields().values())) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below have been inserted.")) - .withViewField(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER).withLabel("# of file rows")) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); - - qInstance.addProcess( - new QProcessMetaData() - .withName(processName) - .withLabel(table.getLabel() + " Bulk Insert") - .withTableName(table.getName()) - .withIsHidden(true) - .withStepList(List.of( - uploadScreen, - receiveFileStep, - reviewScreen, - storeStep, - resultsScreen - ))); + process.addStep(0, uploadScreen); + process.getFrontendStep("review").setRecordListFields(editableFields); + qInstance.addProcess(process); } @@ -364,6 +338,22 @@ public class QInstanceEnricher *******************************************************************************/ private void defineTableBulkEdit(QInstance qInstance, QTableMetaData table, String processName) { + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table.getName()); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); + values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE); + + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( + ExtractViaQueryStep.class, + BulkEditTransformStep.class, + LoadViaUpdateStep.class, + values + ) + .withName(processName) + .withLabel(table.getLabel() + " Bulk Edit") + .withTableName(table.getName()) + .withIsHidden(true); + List editableFields = table.getFields().values().stream() .filter(QFieldMetaData::getIsEditable) .toList(); @@ -381,54 +371,9 @@ public class QInstanceEnricher Fields whose switches are off will not be updated.""")) .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_EDIT_FORM)); - QBackendStepMetaData receiveValuesStep = new QBackendStepMetaData() - .withName("receiveValues") - .withCode(new QCodeReference(BulkEditReceiveValuesStep.class)) - .withInputData(new QFunctionInputMetaData() - .withRecordListMetaData(new QRecordListMetaData().withTableName(table.getName())) - .withField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS, QFieldType.STRING)) - .withFields(editableFields)); - - QFrontendStepMetaData reviewScreen = new QFrontendStepMetaData() - .withName("review") - .withRecordListFields(editableFields) - .withViewField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, QFieldType.STRING)) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below will be updated if you click Submit.")) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); - - QBackendStepMetaData storeStep = new QBackendStepMetaData() - .withName("storeRecords") - .withCode(new QCodeReference(BulkEditStoreRecordsStep.class)) - .withOutputMetaData(new QFunctionOutputMetaData() - .withFieldList(List.of(new QFieldMetaData("noOfFileRows", QFieldType.INTEGER)))); - - QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData() - .withName("results") - .withRecordListFields(new ArrayList<>(table.getFields().values())) - .withViewField(new QFieldMetaData(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, QFieldType.STRING)) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below have been updated.")) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); - - qInstance.addProcess( - new QProcessMetaData() - .withName(processName) - .withLabel(table.getLabel() + " Bulk Edit") - .withTableName(table.getName()) - .withIsHidden(true) - .withStepList(List.of( - LoadInitialRecordsStep.defineMetaData(table.getName()), - editScreen, - receiveValuesStep, - reviewScreen, - storeStep, - resultsScreen - ))); + process.addStep(0, editScreen); + process.getFrontendStep("review").setRecordListFields(editableFields); + qInstance.addProcess(process); } @@ -438,38 +383,26 @@ public class QInstanceEnricher *******************************************************************************/ private void defineTableBulkDelete(QInstance qInstance, QTableMetaData table, String processName) { - QFrontendStepMetaData reviewScreen = new QFrontendStepMetaData() - .withName("review") - .withRecordListFields(new ArrayList<>(table.getFields().values())) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below will be deleted if you click Submit.")) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table.getName()); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); + values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, StreamedETLWithFrontendProcess.DEFAULT_PREVIEW_MESSAGE_FOR_DELETE); - QBackendStepMetaData storeStep = new QBackendStepMetaData() - .withName("delete") - .withCode(new QCodeReference(BulkDeleteStoreStep.class)); + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( + ExtractViaQueryStep.class, + BulkDeleteTransformStep.class, + LoadViaDeleteStep.class, + values + ) + .withName(processName) + .withLabel(table.getLabel() + " Bulk Delete") + .withTableName(table.getName()) + .withIsHidden(true); - QFrontendStepMetaData resultsScreen = new QFrontendStepMetaData() - .withName("results") - .withRecordListFields(new ArrayList<>(table.getFields().values())) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("text", "The records below have been deleted.")) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)); + List tableFields = table.getFields().values().stream().toList(); + process.getFrontendStep("review").setRecordListFields(tableFields); - qInstance.addProcess( - new QProcessMetaData() - .withName(processName) - .withLabel(table.getLabel() + " Bulk Delete") - .withTableName(table.getName()) - .withIsHidden(true) - .withStepList(List.of( - LoadInitialRecordsStep.defineMetaData(table.getName()), - reviewScreen, - storeStep, - resultsScreen - ))); + qInstance.addProcess(process); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java index 7a21349f..12aa596b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java @@ -35,7 +35,7 @@ import java.util.List; public class ProcessSummaryLine implements Serializable { private Status status; - private Integer count; + private Integer count = 0; private String message; ////////////////////////////////////////////////////////////////////////// @@ -77,7 +77,16 @@ public class ProcessSummaryLine implements Serializable { this.status = status; this.message = message; - this.count = 0; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLine(Status status) + { + this.status = status; } @@ -185,12 +194,22 @@ public class ProcessSummaryLine implements Serializable ** *******************************************************************************/ public void incrementCount() + { + incrementCount(1); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void incrementCount(int amount) { if(count == null) { count = 0; } - count++; + count += amount; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java index 40da4720..ec7c1805 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.delete; import java.io.Serializable; import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -35,8 +36,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; *******************************************************************************/ public class DeleteInput extends AbstractTableActionInput { - private List primaryKeys; - private QQueryFilter queryFilter; + private QBackendTransaction transaction; + private List primaryKeys; + private QQueryFilter queryFilter; @@ -59,6 +61,40 @@ public class DeleteInput extends AbstractTableActionInput + /******************************************************************************* + ** Getter for transaction + ** + *******************************************************************************/ + public QBackendTransaction getTransaction() + { + return transaction; + } + + + + /******************************************************************************* + ** Setter for transaction + ** + *******************************************************************************/ + public void setTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + } + + + + /******************************************************************************* + ** Fluent setter for transaction + ** + *******************************************************************************/ + public DeleteInput withTransaction(QBackendTransaction transaction) + { + this.transaction = transaction; + return (this); + } + + + /******************************************************************************* ** Getter for ids ** @@ -92,6 +128,7 @@ public class DeleteInput extends AbstractTableActionInput } + /******************************************************************************* ** Getter for queryFilter ** @@ -113,6 +150,7 @@ public class DeleteInput extends AbstractTableActionInput } + /******************************************************************************* ** Fluent setter for queryFilter ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java deleted file mode 100644 index c8cf5f11..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStep.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete; - - -import java.io.IOException; -import java.io.Serializable; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.JsonUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; - - -/******************************************************************************* - ** Backend step to do a bulk delete. - *******************************************************************************/ -public class BulkDeleteStoreStep implements BackendStep -{ - public static final String ERROR_COUNT = "errorCount"; - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting records..."); - runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal(); - - DeleteInput deleteInput = new DeleteInput(runBackendStepInput.getInstance()); - deleteInput.setSession(runBackendStepInput.getSession()); - deleteInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); - deleteInput.setTableName(runBackendStepInput.getTableName()); - - String queryFilterJSON = runBackendStepInput.getValueString("queryFilterJSON"); - if(StringUtils.hasContent(queryFilterJSON)) - { - try - { - deleteInput.setQueryFilter(JsonUtils.toObject(queryFilterJSON, QQueryFilter.class)); - } - catch(IOException e) - { - throw (new QException("Error loading record query filter from process", e)); - } - } - else if(CollectionUtils.nullSafeHasContents(runBackendStepInput.getRecords())) - { - String primaryKeyField = runBackendStepInput.getTable().getPrimaryKeyField(); - List primaryKeyList = runBackendStepInput.getRecords().stream() - .map(r -> r.getValue(primaryKeyField)) - .toList(); - deleteInput.setPrimaryKeys(primaryKeyList); - } - else - { - throw (new QException("Missing required inputs (queryFilterJSON or record list)")); - } - - DeleteAction deleteAction = new DeleteAction(); - DeleteOutput deleteOutput = deleteAction.execute(deleteInput); - - List recordsWithErrors = Objects.requireNonNullElse(deleteOutput.getRecordsWithErrors(), Collections.emptyList()); - runBackendStepOutput.addValue(ERROR_COUNT, recordsWithErrors.size()); - - runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java new file mode 100644 index 00000000..64c1205a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java @@ -0,0 +1,126 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete; + + +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; + + +/******************************************************************************* + ** Transform step for generic table bulk-insert ETL process + *******************************************************************************/ +public class BulkDeleteTransformStep extends AbstractTransformStep +{ + private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + + private String tableLabel; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + /////////////////////////////////////////////////////// + // capture the table label - for the process summary // + /////////////////////////////////////////////////////// + QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + if(table != null) + { + tableLabel = table.getLabel(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus // + // record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE)) + { + if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " record " + "%,d".formatted(okSummary.getCount())); + } + else + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " record"); + } + } + else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE)) + { + if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting " + tableLabel + " record " + "%,d".formatted(okSummary.getCount())); + } + else + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting " + tableLabel + " records"); + } + } + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // no transformation needs done - just pass records through from input to output, and assume all are OK // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); + okSummary.incrementCount(runBackendStepInput.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(boolean isForResultScreen) + { + if(isForResultScreen) + { + okSummary.setMessage(tableLabel + " records were deleted."); + } + else + { + okSummary.setMessage(tableLabel + " records will be deleted."); + } + + ArrayList rs = new ArrayList<>(); + rs.add(okSummary); + return (rs); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java deleted file mode 100644 index 8253999a..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStep.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; - - -import java.io.Serializable; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; - - -/******************************************************************************* - ** Backend step to receive values for a bulk edit. - *******************************************************************************/ -public class BulkEditReceiveValuesStep implements BackendStep -{ - public static final String FIELD_ENABLED_FIELDS = "bulkEditEnabledFields"; - public static final String FIELD_VALUES_BEING_UPDATED = "valuesBeingUpdated"; - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - String enabledFieldsString = runBackendStepInput.getValueString(FIELD_ENABLED_FIELDS); - String[] enabledFields = enabledFieldsString.split(","); - - //////////////////////////////////////////////////////////////////////////////////////////// - // put the value in all the records (note, this is just for display on the review screen, // - // and/or if we wanted to do some validation - this is NOT what will be store, as the // - // Update action only wants fields that are being changed. // - //////////////////////////////////////////////////////////////////////////////////////////// - for(QRecord record : runBackendStepInput.getRecords()) - { - for(String fieldName : enabledFields) - { - Serializable value = runBackendStepInput.getValue(fieldName); - record.setValue(fieldName, value); - } - } - - BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "will be"); - - runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); - } -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java deleted file mode 100644 index f52a7335..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStep.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; - - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; - - -/******************************************************************************* - ** Backend step to store the records from a bulk insert file - *******************************************************************************/ -public class BulkEditStoreRecordsStep implements BackendStep -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - String enabledFieldsString = runBackendStepInput.getValueString(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS); - String[] enabledFields = enabledFieldsString.split(","); - QTableMetaData table = runBackendStepInput.getTable(); - List recordsToUpdate = new ArrayList<>(); - - runBackendStepInput.getAsyncJobCallback().updateStatus("Updating values in records..."); - int i = 1; - for(QRecord record : runBackendStepInput.getRecords()) - { - runBackendStepInput.getAsyncJobCallback().updateStatus(i++, runBackendStepInput.getRecords().size()); - QRecord recordToUpdate = new QRecord(); - recordsToUpdate.add(recordToUpdate); - - recordToUpdate.setValue(table.getPrimaryKeyField(), record.getValue(table.getPrimaryKeyField())); - for(String fieldName : enabledFields) - { - Serializable value = runBackendStepInput.getValue(fieldName); - recordToUpdate.setValue(fieldName, value); - } - } - - runBackendStepInput.getAsyncJobCallback().updateStatus("Storing updated records..."); - runBackendStepInput.getAsyncJobCallback().clearCurrentAndTotal(); - - UpdateInput updateInput = new UpdateInput(runBackendStepInput.getInstance()); - updateInput.setSession(runBackendStepInput.getSession()); - updateInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback()); - updateInput.setAreAllValuesBeingUpdatedTheSame(true); - updateInput.setTableName(runBackendStepInput.getTableName()); - updateInput.setRecords(recordsToUpdate); - - UpdateAction updateAction = new UpdateAction(); - UpdateOutput updateOutput = updateAction.execute(updateInput); - - runBackendStepOutput.setRecords(updateOutput.getRecords()); - - BulkEditUtils.setFieldValuesBeingUpdated(runBackendStepInput, runBackendStepOutput, enabledFields, "was"); - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java new file mode 100644 index 00000000..954b562c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java @@ -0,0 +1,215 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** Transform step for generic table bulk-edit ETL process + *******************************************************************************/ +public class BulkEditTransformStep extends AbstractTransformStep +{ + public static final String FIELD_ENABLED_FIELDS = "bulkEditEnabledFields"; + + private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + private List infoSummaries = new ArrayList<>(); + + private QTableMetaData table; + private String tableLabel; + private String[] enabledFields; + + private boolean isValidateStep; + private boolean isExecuteStep; + private boolean haveRecordCount; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + /////////////////////////////////////////////////////// + // capture the table label - for the process summary // + /////////////////////////////////////////////////////// + table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + if(table != null) + { + tableLabel = table.getLabel(); + } + + String enabledFieldsString = runBackendStepInput.getValueString(FIELD_ENABLED_FIELDS); + enabledFields = enabledFieldsString.split(","); + + isValidateStep = runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + isExecuteStep = runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE); + haveRecordCount = runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) != null; + + buildInfoSummaryLines(runBackendStepInput, enabledFields); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus // + // record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(isValidateStep) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records"); + if(!haveRecordCount) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Processing record " + "%,d".formatted(okSummary.getCount())); + } + } + else if(isExecuteStep) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Editing " + tableLabel + " records"); + if(!haveRecordCount) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Editing " + tableLabel + " record " + "%,d".formatted(okSummary.getCount())); + } + } + + List outputRecords = new ArrayList<>(); + if(isExecuteStep) + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for the execute step - create new record objects, just with the primary key, and the fields being updated. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : runBackendStepInput.getRecords()) + { + QRecord recordToUpdate = new QRecord(); + recordToUpdate.setValue(table.getPrimaryKeyField(), record.getValue(table.getPrimaryKeyField())); + outputRecords.add(recordToUpdate); + setUpdatedFieldsInRecord(runBackendStepInput, enabledFields, recordToUpdate); + } + } + else + { + //////////////////////////////////////////////////////////////////////////////////////////// + // put the value in all the records (note, this is just for display on the review screen, // + // and/or if we wanted to do some validation - this is NOT what will be store, as the // + // Update action only wants fields that are being changed. // + //////////////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : runBackendStepInput.getRecords()) + { + outputRecords.add(record); + setUpdatedFieldsInRecord(runBackendStepInput, enabledFields, record); + } + } + runBackendStepOutput.setRecords(outputRecords); + okSummary.incrementCount(runBackendStepInput.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void buildInfoSummaryLines(RunBackendStepInput runBackendStepInput, String[] enabledFields) + { + QValueFormatter qValueFormatter = new QValueFormatter(); + for(String fieldName : enabledFields) + { + QFieldMetaData field = table.getField(fieldName); + String label = field.getLabel(); + Serializable value = runBackendStepInput.getValue(fieldName); + + ProcessSummaryLine summaryLine = new ProcessSummaryLine(Status.INFO); + summaryLine.setCount(null); + infoSummaries.add(summaryLine); + + String verb = isExecuteStep ? "was" : "will be"; + if(StringUtils.hasContent(ValueUtils.getValueAsString(value))) + { + String formattedValue = qValueFormatter.formatValue(field, value); // todo - PVS! + summaryLine.setMessage(label + " " + verb + " set to: " + formattedValue); + } + else + { + summaryLine.setMessage(label + " " + verb + " cleared out"); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setUpdatedFieldsInRecord(RunBackendStepInput runBackendStepInput, String[] enabledFields, QRecord record) + { + for(String fieldName : enabledFields) + { + Serializable value = runBackendStepInput.getValue(fieldName); + record.setValue(fieldName, value); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(boolean isForResultScreen) + { + if(isForResultScreen) + { + okSummary.setMessage(tableLabel + " records were edited."); + } + else + { + okSummary.setMessage(tableLabel + " records will be edited."); + } + + ArrayList rs = new ArrayList<>(); + rs.add(okSummary); + rs.addAll(infoSummaries); + return (rs); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java deleted file mode 100644 index 71eeb69f..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditUtils.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; - - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; - - -/******************************************************************************* - ** Utility methods used for Bulk Edit steps - *******************************************************************************/ -public class BulkEditUtils -{ - - /******************************************************************************* - ** - *******************************************************************************/ - public static void setFieldValuesBeingUpdated(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, String[] enabledFields, String verb) - { - ///////////////////////////////////////////////////////////////////// - // build the string to show the user what fields are being changed // - ///////////////////////////////////////////////////////////////////// - List valuesBeingUpdated = new ArrayList<>(); - QTableMetaData table = runBackendStepInput.getTable(); - for(String fieldName : enabledFields) - { - String label = table.getField(fieldName).getLabel(); - Serializable value = runBackendStepInput.getValue(fieldName); - - if(StringUtils.hasContent(ValueUtils.getValueAsString(value))) - { - valuesBeingUpdated.add(label + " " + verb + " set to: " + value); - } - else - { - valuesBeingUpdated.add(label + " " + verb + " cleared out"); - } - } - runBackendStepOutput.addValue(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED, String.join("\n", valuesBeingUpdated)); - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java similarity index 70% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java index 4d4182bb..d8850d40 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java @@ -31,23 +31,22 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; -import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; import com.kingsrook.qqq.backend.core.state.AbstractStateKey; import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; /******************************************************************************* - ** Utility methods used by bulk insert steps + ** Extract step for generic table bulk-insert ETL process *******************************************************************************/ -public class BulkInsertUtils +public class BulkInsertExtractStep extends AbstractExtractStep { - /******************************************************************************* - ** - *******************************************************************************/ - static List getQRecordsFromFile(RunBackendStepInput runBackendStepInput) throws QException + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { AbstractStateKey stateKey = (AbstractStateKey) runBackendStepInput.getValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME); Optional optionalUploadedFile = TempFileStateProvider.getInstance().get(QUploadedFile.class, stateKey); @@ -70,33 +69,35 @@ public class BulkInsertUtils mapping.addMapping(entry.getKey(), entry.getValue().getLabel()); } - List qRecords; + ////////////////////////////////////////////////////////////////////////// + // get the non-editable fields - they'll be blanked out in a customizer // + ////////////////////////////////////////////////////////////////////////// + List nonEditableFields = table.getFields().values().stream() + .filter(f -> !f.getIsEditable()) + .toList(); + if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv")) { - qRecords = new CsvToQRecordAdapter().buildRecordsFromCsv(new String(bytes), runBackendStepInput.getInstance().getTable(tableName), mapping); + new CsvToQRecordAdapter().buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper() + .withRecordPipe(getRecordPipe()) + .withLimit(getLimit()) + .withCsv(new String(bytes)) + .withTable(runBackendStepInput.getInstance().getTable(tableName)) + .withMapping(mapping) + .withRecordCustomizer((record) -> + { + //////////////////////////////////////////// + // remove values from non-editable fields // + //////////////////////////////////////////// + for(QFieldMetaData nonEditableField : nonEditableFields) + { + record.setValue(nonEditableField.getName(), null); + } + })); } else { throw (new QUserFacingException("Unsupported file type.")); } - - //////////////////////////////////////////////// - // remove values from any non-editable fields // - //////////////////////////////////////////////// - List nonEditableFields = table.getFields().values().stream() - .filter(f -> !f.getIsEditable()) - .toList(); - if(!nonEditableFields.isEmpty()) - { - for(QRecord qRecord : qRecords) - { - for(QFieldMetaData nonEditableField : nonEditableFields) - { - qRecord.setValue(nonEditableField.getName(), null); - } - } - } - - return (qRecords); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java deleted file mode 100644 index 7844ef9c..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; - - -/******************************************************************************* - ** Backend step to receive a bulk-insert upload file - *******************************************************************************/ -public class BulkInsertReceiveFileStep implements BackendStep -{ - /******************************************************************************* - ** - *******************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - List qRecords = BulkInsertUtils.getQRecordsFromFile(runBackendStepInput); - - runBackendStepOutput.addValue("noOfFileRows", qRecords.size()); - runBackendStepOutput.setRecords(qRecords); - } -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java new file mode 100644 index 00000000..3c0fabf3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -0,0 +1,119 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; + + +/******************************************************************************* + ** Transform step for generic table bulk-insert ETL process + *******************************************************************************/ +public class BulkInsertTransformStep extends AbstractTransformStep +{ + private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + + private String tableLabel; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + /////////////////////////////////////////////////////// + // capture the table label - for the process summary // + /////////////////////////////////////////////////////// + QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + if(table != null) + { + tableLabel = table.getLabel(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus // + // record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE)) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Processing row " + "%,d".formatted(okSummary.getCount())); + } + else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE)) + { + if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null) + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting " + tableLabel + " record " + "%,d".formatted(okSummary.getCount())); + } + else + { + runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting " + tableLabel + " records"); + } + } + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // no transformation needs done - just pass records through from input to output, and assume all are OK // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); + okSummary.incrementCount(runBackendStepInput.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(boolean isForResultScreen) + { + if(isForResultScreen) + { + okSummary.setMessage(tableLabel + " records were inserted."); + } + else + { + okSummary.setMessage(tableLabel + " records will be inserted."); + } + + ArrayList rs = new ArrayList<>(); + rs.add(okSummary); + return (rs); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java index 888857d8..a7a86049 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java @@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.io.IOException; import java.io.Serializable; +import java.util.Arrays; +import java.util.List; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -31,9 +33,13 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -94,10 +100,12 @@ public class ExtractViaQueryStep extends AbstractExtractStep *******************************************************************************/ protected QQueryFilter getQueryFilter(RunBackendStepInput runBackendStepInput) throws QException { + String queryFilterJson = runBackendStepInput.getValueString("queryFilterJson"); + Serializable defaultQueryFilter = runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER); + ////////////////////////////////////////////////////////////////////////////////////// // if the queryFilterJson field is populated, read the filter from it and return it // ////////////////////////////////////////////////////////////////////////////////////// - String queryFilterJson = runBackendStepInput.getValueString("queryFilterJson"); if(queryFilterJson != null) { return getQueryFilterFromJson(queryFilterJson, "Error loading query filter from json field"); @@ -111,20 +119,41 @@ public class ExtractViaQueryStep extends AbstractExtractStep runBackendStepInput.addValue("queryFilterJson", JsonUtils.toJson(queryFilter)); return (queryFilter); } - else + else if(defaultQueryFilter instanceof QQueryFilter filter) { - ///////////////////////////////////////////////////// - // else, see if a defaultQueryFilter was specified // - ///////////////////////////////////////////////////// - Serializable defaultQueryFilter = runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER); - if(defaultQueryFilter instanceof QQueryFilter filter) + ///////////////////////////////////////////////////////////////////////////// + // else, see if a defaultQueryFilter was specified as a QueryFilter object // + ///////////////////////////////////////////////////////////////////////////// + return (filter); + } + else if(defaultQueryFilter instanceof String string) + { + ///////////////////////////////////////////////////////////////////////////// + // else, see if a defaultQueryFilter was specified as a JSON string + ///////////////////////////////////////////////////////////////////////////// + return getQueryFilterFromJson(string, "Error loading default query filter from json"); + } + else if(StringUtils.hasContent(runBackendStepInput.getValueString("filterJSON"))) + { + /////////////////////////////////////////////////////////////////////// + // else, check for filterJSON from a frontend launching of a process // + /////////////////////////////////////////////////////////////////////// + return getQueryFilterFromJson(runBackendStepInput.getValueString("filterJSON"), "Error loading default query filter from json"); + } + else if(StringUtils.hasContent(runBackendStepInput.getValueString("recordIds"))) + { + ////////////////////////////////////////////////////////////////////// + // else, check for recordIds from a frontend launching of a process // + ////////////////////////////////////////////////////////////////////// + QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE)); + if(table == null) { - return (filter); - } - else if(defaultQueryFilter instanceof String string) - { - return getQueryFilterFromJson(string, "Error loading default query filter from json"); + throw (new QException("source table name was not set - could not load records by id")); } + String recordIds = runBackendStepInput.getValueString("recordIds"); + Serializable[] split = recordIds.split(","); + List idStrings = Arrays.stream(split).toList(); + return (new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, idStrings))); } throw (new QException("Could not find query filter for Extract step.")); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java similarity index 51% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStep.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java index 1f6710d5..34aa0d3b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java @@ -19,41 +19,62 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; -import java.util.List; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import java.util.Optional; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; /******************************************************************************* - ** Backend step to store the records from a bulk insert file + ** Generic implementation of a LoadStep - that runs a Delete action for a + ** specified table. *******************************************************************************/ -public class BulkInsertStoreRecordsStep implements BackendStep +public class LoadViaDeleteStep extends AbstractLoadStep { + public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. ** *******************************************************************************/ @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - List qRecords = BulkInsertUtils.getQRecordsFromFile(runBackendStepInput); + QTableMetaData table = runBackendStepInput.getTable(); + DeleteInput deleteInput = new DeleteInput(runBackendStepInput.getInstance()); + deleteInput.setSession(runBackendStepInput.getSession()); + deleteInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE)); + deleteInput.setPrimaryKeys(runBackendStepInput.getRecords().stream().map(r -> r.getValue(table.getPrimaryKeyField())).collect(Collectors.toList())); + // todo? can make more efficient deletes, maybe? deleteInput.setQueryFilter(); + getTransaction().ifPresent(deleteInput::setTransaction); + new DeleteAction().execute(deleteInput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Optional openTransaction(RunBackendStepInput runBackendStepInput) throws QException + { InsertInput insertInput = new InsertInput(runBackendStepInput.getInstance()); insertInput.setSession(runBackendStepInput.getSession()); - insertInput.setTableName(runBackendStepInput.getTableName()); - insertInput.setRecords(qRecords); + insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE)); - InsertAction insertAction = new InsertAction(); - InsertOutput insertOutput = insertAction.execute(insertInput); - - runBackendStepOutput.setRecords(insertOutput.getRecords()); + return (Optional.of(new InsertAction().openTransaction(insertInput))); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 5d872b7c..9b982921 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -82,6 +82,7 @@ public class StreamedETLWithFrontendProcess public static final String DEFAULT_PREVIEW_MESSAGE_FOR_INSERT = "This is a preview of the records that will be created."; public static final String DEFAULT_PREVIEW_MESSAGE_FOR_UPDATE = "This is a preview of the records that will be updated."; + public static final String DEFAULT_PREVIEW_MESSAGE_FOR_DELETE = "This is a preview of the records that will be deleted."; public static final String FIELD_PREVIEW_MESSAGE = "previewMessage"; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java deleted file mode 100644 index 483e2edf..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteStoreStepTest.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete; - - -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.utils.JsonUtils; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - - -/******************************************************************************* - ** Unit test for BulkDeleteStoreStep - *******************************************************************************/ -class BulkDeleteStoreStepTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testWithoutFilter() throws QException - { - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.setRecords(TestUtils.queryTable(TestUtils.defineTablePerson().getName())); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - new BulkDeleteStoreStep().run(stepInput, stepOutput); - assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT)); - } - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testWithFilter() throws QException - { - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue("queryFilterJSON", JsonUtils.toJson(new QQueryFilter())); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - new BulkDeleteStoreStep().run(stepInput, stepOutput); - assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT)); - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTest.java new file mode 100644 index 00000000..1ede46da --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTest.java @@ -0,0 +1,119 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for full bulk edit process + *******************************************************************************/ +class BulkDeleteTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + ////////////////////////////// + // insert some test records // + ////////////////////////////// + QInstance qInstance = TestUtils.defineInstance(); + TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Darin").withValue("lastName", "Kelkhoff").withValue("email", "darin.kelkhoff@kingsrook.com"), + new QRecord().withValue("id", 2).withValue("firstName", "Tim").withValue("lastName", "Chamberlain").withValue("email", "tim.chamberlain@kingsrook.com"), + new QRecord().withValue("id", 3).withValue("firstName", "James").withValue("lastName", "Maes").withValue("email", "james.maes@kingsrook.com") + )); + + ////////////////////////////////// + // set up the run-process input // + ////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(qInstance); + runProcessInput.setSession(TestUtils.getMockSession()); + runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkDelete"); + runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, + new QQueryFilter().withCriteria(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 3)))); + + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + + runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + runProcessInput.addValue("processUUID", processUUID); + runProcessInput.setStartAfterStep("review"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class); + + runProcessInput.setStartAfterStep("review"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); + assertThat(runProcessOutput.getException()).isEmpty(); + + @SuppressWarnings("unchecked") + List processSummaryLines = (List) runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY); + assertThat(processSummaryLines).hasSize(1); + assertThat(processSummaryLines.stream().filter(psl -> psl.getStatus().equals(Status.OK))).hasSize(1); + + ////////////////////////////////////////////////////////////// + // query for the records - assert that only id=2 exists now // + ////////////////////////////////////////////////////////////// + List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + assertEquals(2, records.get(0).getValueInteger("id")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStepTest.java deleted file mode 100644 index 0619d13a..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditReceiveValuesStepTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - - -/******************************************************************************* - ** Unit test for BulkEditReceiveValuesStep - *******************************************************************************/ -class BulkEditReceiveValuesStepTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void test() throws QException - { - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS, "firstName,email,birthDate"); - stepInput.addValue("firstName", "Johnny"); - stepInput.addValue("email", null); - stepInput.addValue("birthDate", "1909-01-09"); - List records = TestUtils.queryTable(TestUtils.defineTablePerson().getName()); - stepInput.setRecords(records); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - new BulkEditReceiveValuesStep().run(stepInput, stepOutput); - - String valuesBeingUpdated = stepOutput.getValueString(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED); - assertThat(valuesBeingUpdated).matches("(?s).*First Name.*Johnny.*"); - assertThat(valuesBeingUpdated).matches("(?s).*Email will be cleared.*"); - assertThat(valuesBeingUpdated).matches("(?s).*Birth Date.*1909-01-09.*"); - - int count = 0; - for(QRecord record : stepOutput.getRecords()) - { - assertEquals("Johnny", record.getValueString("firstName")); - assertNull(record.getValue("email")); - // todo value utils needed in getValueDate... assertEquals(LocalDate.of(1909, 1, 9), record.getValueDate("birthDate")); - count++; - } - assertEquals(records.size(), count); - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStepTest.java deleted file mode 100644 index 9b110db4..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditStoreRecordsStepTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - - -/******************************************************************************* - ** Unit test for BulkEditStoreRecordsStep - *******************************************************************************/ -class BulkEditStoreRecordsStepTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void test() throws QException - { - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS, "firstName,email,birthDate"); - stepInput.addValue("firstName", "Johnny"); - stepInput.addValue("email", null); - stepInput.addValue("birthDate", "1909-01-09"); - List records = TestUtils.queryTable(TestUtils.defineTablePerson().getName()); - stepInput.setRecords(records); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - new BulkEditStoreRecordsStep().run(stepInput, stepOutput); - - assertRecordValues(stepOutput.getRecords()); - - // re-fetch the records, make sure they are updated. - // but since Mock backend doesn't actually update them, we can't do this.. - // todo - implement an in-memory backend, that would do this. - // List updatedRecords = TestUtils.queryTable(TestUtils.defineTablePerson().getName()); - // assertRecordValues(updatedRecords); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void assertRecordValues(List records) - { - for(QRecord record : records) - { - assertEquals("Johnny", record.getValueString("firstName")); - assertNull(record.getValue("email")); - // todo value utils needed in getValueDate... assertEquals(LocalDate.of(1909, 1, 9), record.getValueDate("birthDate")); - } - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTest.java new file mode 100644 index 00000000..e1b666d7 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTest.java @@ -0,0 +1,145 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit; + + +import java.util.List; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for full bulk edit process + *******************************************************************************/ +class BulkEditTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + ////////////////////////////// + // insert some test records // + ////////////////////////////// + QInstance qInstance = TestUtils.defineInstance(); + TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Darin").withValue("lastName", "Kelkhoff").withValue("email", "darin.kelkhoff@kingsrook.com"), + new QRecord().withValue("id", 2).withValue("firstName", "Tim").withValue("lastName", "Chamberlain").withValue("email", "tim.chamberlain@kingsrook.com"), + new QRecord().withValue("id", 3).withValue("firstName", "James").withValue("lastName", "Maes").withValue("email", "james.maes@kingsrook.com") + )); + + ////////////////////////////////// + // set up the run-process input // + ////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(qInstance); + runProcessInput.setSession(TestUtils.getMockSession()); + runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkEdit"); + runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, + new QQueryFilter().withCriteria(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 2)))); + + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + + runProcessInput.addValue(BulkEditTransformStep.FIELD_ENABLED_FIELDS, "firstName,email,birthDate"); + runProcessInput.addValue("firstName", "Johnny"); + runProcessInput.addValue("email", null); + runProcessInput.addValue("birthDate", "1909-01-09"); + + runProcessInput.setProcessUUID(processUUID); + runProcessInput.setStartAfterStep("edit"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + + runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + runProcessInput.setStartAfterStep("review"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class); + + runProcessInput.setStartAfterStep("review"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); + assertThat(runProcessOutput.getException()).isEmpty(); + + @SuppressWarnings("unchecked") + List processSummaryLines = (List) runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY); + assertThat(processSummaryLines).hasSize(4); + assertThat(processSummaryLines.stream().filter(psl -> psl.getStatus().equals(Status.OK))).hasSize(1); + List infoLines = processSummaryLines.stream().filter(psl -> psl.getStatus().equals(Status.INFO)).collect(Collectors.toList()); + assertThat(infoLines).hasSize(3); + assertThat(infoLines.stream().map(ProcessSummaryLine::getMessage)).anyMatch(m -> m.matches("(?s).*First Name.*Johnny.*")); + assertThat(infoLines.stream().map(ProcessSummaryLine::getMessage)).anyMatch(m -> m.matches("(?s).*Email was cleared.*")); + assertThat(infoLines.stream().map(ProcessSummaryLine::getMessage)).anyMatch(m -> m.matches("(?s).*Birth Date.*1909-01-09.*")); + + ///////////////////////////////////////////////////////////////////////////////// + // query for the edited records - assert that id 1 & 2 were updated, 3 was not // + ///////////////////////////////////////////////////////////////////////////////// + List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + assertEquals("Johnny", records.get(0).getValueString("firstName")); + assertEquals("Johnny", records.get(1).getValueString("firstName")); + assertEquals("James", records.get(2).getValueString("firstName")); + assertEquals("1909-01-09", records.get(0).getValueString("birthDate")); + assertEquals("1909-01-09", records.get(1).getValueString("birthDate")); + assertNull(records.get(2).getValueString("birthDate")); + assertNull(records.get(0).getValue("email")); + assertNull(records.get(1).getValue("email")); + assertEquals("james.maes@kingsrook.com", records.get(2).getValue("email")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java deleted file mode 100644 index 1b6d5659..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStepTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; -import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.state.StateType; -import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; -import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - - -/******************************************************************************* - ** Unit test for BulkInsertReceiveFileStep - *******************************************************************************/ -class BulkInsertReceiveFileStepTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void test() throws QException - { - //////////////////////////////////////////////////////////////// - // create an uploaded file, similar to how an http server may // - //////////////////////////////////////////////////////////////// - QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); - qUploadedFile.setFilename("test.csv"); - UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); - TempFileStateProvider.getInstance().put(key, qUploadedFile); - - //////////////////////////// - // setup and run the step // - //////////////////////////// - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - new BulkInsertReceiveFileStep().run(stepInput, stepOutput); - - List records = stepOutput.getRecords(); - assertEquals(2, records.size()); - assertEquals("John", records.get(0).getValueString("firstName")); - assertEquals("Jane", records.get(1).getValueString("firstName")); - assertNull(records.get(0).getValue("id")); - assertNull(records.get(1).getValue("id")); - - assertEquals(2, stepOutput.getValueInteger("noOfFileRows")); - } - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testBadFileType() throws QException - { - //////////////////////////////////////////////////////////////// - // create an uploaded file, similar to how an http server may // - //////////////////////////////////////////////////////////////// - QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); // todo - this is NOT excel content... - qUploadedFile.setFilename("test.xslx"); - UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); - TempFileStateProvider.getInstance().put(key, qUploadedFile); - - //////////////////////////// - // setup and run the step // - //////////////////////////// - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - - assertThrows(QUserFacingException.class, () -> - { - new BulkInsertReceiveFileStep().run(stepInput, stepOutput); - }); - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java deleted file mode 100644 index 2853abfc..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStoreRecordsStepTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.state.StateType; -import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; -import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - - -/******************************************************************************* - ** Unit test for BulkInsertStoreRecordsStep - *******************************************************************************/ -class BulkInsertStoreRecordsStepTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void test() throws QException - { - //////////////////////////////////////////////////////////////// - // create an uploaded file, similar to how an http server may // - //////////////////////////////////////////////////////////////// - QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); - qUploadedFile.setFilename("test.csv"); - UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); - TempFileStateProvider.getInstance().put(key, qUploadedFile); - - //////////////////////////// - // setup and run the step // - //////////////////////////// - RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance()); - stepInput.setSession(TestUtils.getMockSession()); - stepInput.setTableName(TestUtils.defineTablePerson().getName()); - stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key); - - RunBackendStepOutput stepOutput = new RunBackendStepOutput(); - new BulkInsertStoreRecordsStep().run(stepInput, stepOutput); - - List records = stepOutput.getRecords(); - assertEquals(2, records.size()); - assertEquals("John", records.get(0).getValueString("firstName")); - assertEquals("Jane", records.get(1).getValueString("firstName")); - assertNotNull(records.get(0).getValue("id")); - assertNotNull(records.get(1).getValue("id")); - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java new file mode 100644 index 00000000..c632ad0f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.state.StateType; +import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; +import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for full bulk insert process + *******************************************************************************/ +class BulkInsertTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + /////////////////////////////////////// + // make sure table is empty to start // + /////////////////////////////////////// + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); + + //////////////////////////////////////////////////////////////// + // create an uploaded file, similar to how an http server may // + //////////////////////////////////////////////////////////////// + QUploadedFile qUploadedFile = new QUploadedFile(); + qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); + qUploadedFile.setFilename("test.csv"); + UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); + TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile); + + RunProcessInput runProcessInput = new RunProcessInput(TestUtils.defineInstance()); + runProcessInput.setSession(TestUtils.getMockSession()); + runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkInsert"); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + + runProcessInput.setProcessUUID(processUUID); + runProcessInput.setStartAfterStep("upload"); + runProcessInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, uploadedFileKey); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + + runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + runProcessInput.setStartAfterStep("review"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class); + + runProcessInput.setStartAfterStep("review"); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); + assertThat(runProcessOutput.getException()).isEmpty(); + + //////////////////////////////////// + // query for the inserted records // + //////////////////////////////////// + List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + assertEquals("John", records.get(0).getValueString("firstName")); + assertEquals("Jane", records.get(1).getValueString("firstName")); + assertNotNull(records.get(0).getValue("id")); + assertNotNull(records.get(1).getValue("id")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 80fc9cd6..a29afce5 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarChart; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge; @@ -164,7 +163,6 @@ public class TestUtils - /******************************************************************************* ** *******************************************************************************/ @@ -207,6 +205,20 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static void insertRecords(QInstance qInstance, QTableMetaData table, List records) throws QException + { + InsertInput insertInput = new InsertInput(qInstance); + insertInput.setSession(new QSession()); + insertInput.setTableName(table.getName()); + insertInput.setRecords(records); + new InsertAction().execute(insertInput); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java index feb81cd1..4e5a8e7f 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java @@ -75,33 +75,56 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte // - else if there's a list, try to delete it, but upon error: // // - - do a single-delete for each entry in the list. // ///////////////////////////////////////////////////////////////////////////////// - try(Connection connection = getConnection(deleteInput)) + try { - /////////////////////////////////////////////////////////////////////////////////////////////// - // if there's a query filter, try to do a single-delete with that filter in the WHERE clause // - /////////////////////////////////////////////////////////////////////////////////////////////// - if(deleteInput.getQueryFilter() != null) + Connection connection; + boolean needToCloseConnection = false; + if(deleteInput.getTransaction() != null && deleteInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction) { - try - { - deleteInput.getAsyncJobCallback().updateStatus("Running bulk delete via query filter."); - deleteQueryFilter(connection, deleteInput, deleteOutput); - return (deleteOutput); - } - catch(Exception e) - { - deleteInput.getAsyncJobCallback().updateStatus("Error running bulk delete via filter. Fetching keys for individual deletes."); - LOG.info("Exception trying to delete by filter query. Moving on to deleting by id now."); - deleteInput.setPrimaryKeys(DeleteAction.getPrimaryKeysFromQueryFilter(deleteInput)); - } + LOG.debug("Using connection from updateInput [" + rdbmsTransaction.getConnection() + "]"); + connection = rdbmsTransaction.getConnection(); + } + else + { + connection = getConnection(deleteInput); + needToCloseConnection = true; } - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // at this point, there either wasn't a query filter, or there was an error executing it (in which case, the query should // - // have been converted to a list of primary keys in the deleteInput). so, proceed now by deleting a list of pkeys. // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - deleteList(connection, deleteInput, deleteOutput); - return (deleteOutput); + try + { + /////////////////////////////////////////////////////////////////////////////////////////////// + // if there's a query filter, try to do a single-delete with that filter in the WHERE clause // + /////////////////////////////////////////////////////////////////////////////////////////////// + if(deleteInput.getQueryFilter() != null) + { + try + { + deleteInput.getAsyncJobCallback().updateStatus("Running bulk delete via query filter."); + deleteQueryFilter(connection, deleteInput, deleteOutput); + return (deleteOutput); + } + catch(Exception e) + { + deleteInput.getAsyncJobCallback().updateStatus("Error running bulk delete via filter. Fetching keys for individual deletes."); + LOG.info("Exception trying to delete by filter query. Moving on to deleting by id now."); + deleteInput.setPrimaryKeys(DeleteAction.getPrimaryKeysFromQueryFilter(deleteInput)); + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // at this point, there either wasn't a query filter, or there was an error executing it (in which case, the query should // + // have been converted to a list of primary keys in the deleteInput). so, proceed now by deleting a list of pkeys. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + deleteList(connection, deleteInput, deleteOutput); + return (deleteOutput); + } + finally + { + if(needToCloseConnection) + { + connection.close(); + } + } } catch(Exception e) { @@ -117,6 +140,13 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte private void deleteList(Connection connection, DeleteInput deleteInput, DeleteOutput deleteOutput) { List primaryKeys = deleteInput.getPrimaryKeys(); + if(primaryKeys.size() == 0) + { + ///////////////////////// + // noop - just return. // + ///////////////////////// + return; + } if(primaryKeys.size() == 1) { doDeleteOne(connection, deleteInput.getTable(), primaryKeys.get(0), deleteOutput); From 0d0a7aec1c52e7cabcfacf2561e63fad7e714a80 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 8 Sep 2022 10:58:21 -0500 Subject: [PATCH 28/28] Initial checkin --- .../QuickSightChartRendererTest.java | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRendererTest.java diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRendererTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRendererTest.java new file mode 100644 index 00000000..34987950 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRendererTest.java @@ -0,0 +1,80 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard; + + +import java.net.UnknownHostException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QuickSightChartMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Unit test for QuickSightChartRenderer + *******************************************************************************/ +class QuickSightChartRendererTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWrongMetaDataClass() throws QException + { + assertThatThrownBy(() -> new QuickSightChartRenderer().render(TestUtils.defineInstance(), TestUtils.getMockSession(), new QWidgetMetaData())) + .hasRootCauseInstanceOf(ClassCastException.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNoCredentials() throws QException + { + assertThatThrownBy(() -> new QuickSightChartRenderer().render(TestUtils.defineInstance(), TestUtils.getMockSession(), new QuickSightChartMetaData())) + .hasRootCauseInstanceOf(NullPointerException.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadCredentials() throws QException + { + assertThatThrownBy(() -> new QuickSightChartRenderer().render(TestUtils.defineInstance(), TestUtils.getMockSession(), + new QuickSightChartMetaData() + .withName("test") + .withAccessKey("FAIL") + .withSecretKey("FAIL") + .withRegion("FAIL") + .withAccountId("FAIL") + )).hasRootCauseInstanceOf(UnknownHostException.class); + } + +} \ No newline at end of file