Feedback from code reviews

checkstyle/config.xml

@@ -261,5 +261,9 @@
       <module name="MissingOverride"/>
 
    </module>
+   <module name="Header">
+     <property name="headerFile" value="checkstyle/license.txt"/>
+     <property name="fileExtensions" value="java"/>
+   </module>
    <module name="SuppressWarningsFilter"/>
 </module>

checkstyle/license.txt

@@ -0,0 +1,20 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */

pom.xml

@@ -122,7 +122,8 @@
                   <id>validate</id>
                   <phase>validate</phase>
                   <configuration>
-                     <configLocation>checkstyle.xml</configLocation>
+                     <configLocation>checkstyle/config.xml</configLocation>
+                     <headerLocation>checkstyle/license.txt</headerLocation>
                      <!-- <suppressionsLocation>checkstyle-suppressions.xml</suppressionsLocation> -->
                      <encoding>UTF-8</encoding>
                      <consoleOutput>true</consoleOutput>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobCallback.java

@@ -50,6 +50,17 @@ public class AsyncJobCallback
 
 
 
+   /*******************************************************************************
+    ** Setter for asyncJobStatus
+    **
+    *******************************************************************************/
+   public void setAsyncJobStatus(AsyncJobStatus asyncJobStatus)
+   {
+      this.asyncJobStatus = asyncJobStatus;
+   }
+
+
+
    /*******************************************************************************
     ** Update the message
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.async;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationHandler.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 
@@ -46,6 +67,13 @@ public class RecordAutomationStatusUpdater
          automationStatus = AutomationStatus.OK;
       }
 
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // In case an automation is running, and it updates records - don't let those records be marked  //
+      // as PENDING_UPDATE_AUTOMATIONS... this is meant to avoid having a record's automation update   //
+      // itself, and then continue to do so in a loop (infinitely).                                    //
+      // BUT - shouldn't this be allowed to update OTHER records to be pending updates?  It seems like //
+      // yes - so -that'll probably be a bug to fix at some point in the future todo                   //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
       if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
       {
          Exception e = new Exception();
@@ -86,17 +114,11 @@ public class RecordAutomationStatusUpdater
 
       if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS))
       {
-         if(tableActions.stream().noneMatch(a -> TriggerEvent.POST_INSERT.equals(a.getTriggerEvent())))
+         return tableActions.stream().noneMatch(a -> TriggerEvent.POST_INSERT.equals(a.getTriggerEvent()));
-         {
-            return (true);
-         }
       }
       else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
       {
-         if(tableActions.stream().noneMatch(a -> TriggerEvent.POST_UPDATE.equals(a.getTriggerEvent())))
+         return tableActions.stream().noneMatch(a -> TriggerEvent.POST_UPDATE.equals(a.getTriggerEvent()));
-         {
-            return (true);
-         }
       }
 
       return (false);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutor.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.automation.polling;
 
 
@@ -82,8 +103,7 @@ public class PollingAutomationExecutor
     *******************************************************************************/
    public void stopAsync()
    {
-      Runnable stopper = this::stop;
+      new Thread(this::stop).start();
-      stopper.run();
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunner.java

@@ -1,8 +1,30 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.automation.polling;
 
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
@@ -14,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
 import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
 import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -28,14 +51,14 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import org.apache.commons.lang.NotImplementedException;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -57,6 +80,23 @@ class PollingAutomationRunner implements Runnable
    private Map<String, List<TableAutomationAction>> tableInsertActions = new HashMap<>();
    private Map<String, List<TableAutomationAction>> tableUpdateActions = new HashMap<>();
 
+   private Map<String, Map<AutomationStatus, List<TableAutomationAction>>> tableActions = new HashMap<>();
+
+   private static Map<TriggerEvent, AutomationStatus> triggerEventAutomationStatusMap = Map.of(
+      TriggerEvent.POST_INSERT, AutomationStatus.PENDING_INSERT_AUTOMATIONS,
+      TriggerEvent.POST_UPDATE, AutomationStatus.PENDING_UPDATE_AUTOMATIONS
+   );
+
+   private static Map<AutomationStatus, AutomationStatus> pendingToRunningStatusMap = Map.of(
+      AutomationStatus.PENDING_INSERT_AUTOMATIONS, AutomationStatus.RUNNING_INSERT_AUTOMATIONS,
+      AutomationStatus.PENDING_UPDATE_AUTOMATIONS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS
+   );
+
+   private static Map<AutomationStatus, AutomationStatus> pendingToFailedStatusMap = Map.of(
+      AutomationStatus.PENDING_INSERT_AUTOMATIONS, AutomationStatus.FAILED_INSERT_AUTOMATIONS,
+      AutomationStatus.PENDING_UPDATE_AUTOMATIONS, AutomationStatus.FAILED_UPDATE_AUTOMATIONS
+   );
+
 
 
    /*******************************************************************************
@@ -83,16 +123,10 @@ class PollingAutomationRunner implements Runnable
             ///////////////////////////////////////////////////////////////////////////
             for(TableAutomationAction action : table.getAutomationDetails().getActions())
             {
-               if(TriggerEvent.POST_INSERT.equals(action.getTriggerEvent()))
+               AutomationStatus automationStatus = triggerEventAutomationStatusMap.get(action.getTriggerEvent());
-               {
+               tableActions.putIfAbsent(table.getName(), new HashMap<>());
-                  tableInsertActions.putIfAbsent(table.getName(), new ArrayList<>());
+               tableActions.get(table.getName()).putIfAbsent(automationStatus, new ArrayList<>());
-                  tableInsertActions.get(table.getName()).add(action);
+               tableActions.get(table.getName()).get(automationStatus).add(action);
-               }
-               else if(TriggerEvent.POST_UPDATE.equals(action.getTriggerEvent()))
-               {
-                  tableUpdateActions.putIfAbsent(table.getName(), new ArrayList<>());
-                  tableUpdateActions.get(table.getName()).add(action);
-               }
             }
 
             //////////////////////////////
@@ -143,8 +177,8 @@ class PollingAutomationRunner implements Runnable
    private void processTable(QTableMetaData table) throws QException
    {
       QSession session = sessionSupplier != null ? sessionSupplier.get() : new QSession();
-      processTableInsertOrUpdate(table, session, true);
+      processTableInsertOrUpdate(table, session, AutomationStatus.PENDING_INSERT_AUTOMATIONS);
-      processTableInsertOrUpdate(table, session, false);
+      processTableInsertOrUpdate(table, session, AutomationStatus.PENDING_UPDATE_AUTOMATIONS);
    }
 
 
@@ -152,10 +186,11 @@ class PollingAutomationRunner implements Runnable
    /*******************************************************************************
     ** Query for and process records that have a PENDING_INSERT or PENDING_UPDATE status on a given table.
     *******************************************************************************/
-   private void processTableInsertOrUpdate(QTableMetaData table, QSession session, boolean isInsert) throws QException
+   private void processTableInsertOrUpdate(QTableMetaData table, QSession session, AutomationStatus automationStatus) throws QException
    {
-      AutomationStatus            automationStatus = isInsert ? AutomationStatus.PENDING_INSERT_AUTOMATIONS : AutomationStatus.PENDING_UPDATE_AUTOMATIONS;
+      List<TableAutomationAction> actions = tableActions
-      List<TableAutomationAction> actions          = (isInsert ? tableInsertActions : tableUpdateActions).get(table.getName());
+         .getOrDefault(table.getName(), Collections.emptyMap())
+         .getOrDefault(automationStatus, Collections.emptyList());
       if(CollectionUtils.nullSafeIsEmpty(actions))
       {
          return;
@@ -168,18 +203,28 @@ class PollingAutomationRunner implements Runnable
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       RecordPipe          recordPipe          = new RecordPipe();
       AsyncRecordPipeLoop asyncRecordPipeLoop = new AsyncRecordPipeLoop();
-      asyncRecordPipeLoop.run("PollingAutomationRunner>Query>" + (isInsert ? "insert" : "update"), null, recordPipe, (status) ->
+      asyncRecordPipeLoop.run("PollingAutomationRunner>Query>" + automationStatus, null, recordPipe, (status) ->
          {
             QueryInput queryInput = new QueryInput(instance);
             queryInput.setSession(session);
             queryInput.setTableName(table.getName());
-            queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(table.getAutomationDetails().getStatusTracking().getFieldName(), QCriteriaOperator.IN, List.of(automationStatus.getId()))));
+
+            AutomationStatusTrackingType statusTrackingType = table.getAutomationDetails().getStatusTracking().getType();
+            if(AutomationStatusTrackingType.FIELD_IN_TABLE.equals(statusTrackingType))
+            {
+               queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(table.getAutomationDetails().getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId()))));
+            }
+            else
+            {
+               throw (new NotImplementedException("Automation Status Tracking type [" + statusTrackingType + "] is not yet implemented in here."));
+            }
+
             queryInput.setRecordPipe(recordPipe);
             return (new QueryAction().execute(queryInput));
          }, () ->
          {
             List<QRecord> records = recordPipe.consumeAvailableRecords();
-            applyActionsToRecords(session, table, records, actions, isInsert);
+            applyActionsToRecords(session, table, records, actions, automationStatus);
             return (records.size());
          }
       );
@@ -189,9 +234,10 @@ class PollingAutomationRunner implements Runnable
 
    /*******************************************************************************
     ** For a set of records that were found to be in a PENDING state - run all the
-    ** table's actions against them.
+    ** table's actions against them - IF they are found to match the action's filter
+    ** (assuming it has one - if it doesn't, then all records match).
     *******************************************************************************/
-   private void applyActionsToRecords(QSession session, QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, boolean isInsert) throws QException
+   private void applyActionsToRecords(QSession session, QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, AutomationStatus automationStatus) throws QException
    {
       if(CollectionUtils.nullSafeIsEmpty(records))
       {
@@ -201,7 +247,7 @@ class PollingAutomationRunner implements Runnable
       ///////////////////////////////////////////////////
       // mark the records as RUNNING their automations //
       ///////////////////////////////////////////////////
-      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, isInsert ? AutomationStatus.RUNNING_INSERT_AUTOMATIONS : AutomationStatus.RUNNING_UPDATE_AUTOMATIONS);
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToRunningStatusMap.get(automationStatus));
 
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // foreach action - run it against the records (but only if they match the action's filter, if there is one) //
@@ -234,7 +280,7 @@ class PollingAutomationRunner implements Runnable
       ////////////////////////////////////////
       if(anyActionsFailed)
       {
-         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.FAILED_UPDATE_AUTOMATIONS);
+         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToFailedStatusMap.get(automationStatus));
       }
       else
       {
@@ -302,25 +348,24 @@ class PollingAutomationRunner implements Runnable
    {
       if(StringUtils.hasContent(action.getProcessName()))
       {
+         /////////////////////////////////////////////////////////////////////////////////////////
+         // if the action has a process associated with it - run that process.                  //
+         // tell it to SKIP frontend steps.                                                     //
+         // give the process a callback w/ a query filter that has the p-keys of these records. //
+         /////////////////////////////////////////////////////////////////////////////////////////
          RunProcessInput runProcessInput = new RunProcessInput(instance);
          runProcessInput.setSession(session);
          runProcessInput.setProcessName(action.getProcessName());
          runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
-
+         runProcessInput.setCallback(new QProcessCallback()
-         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         {
-         // kinda hacky - if we see that this process has an input field of a given name, then put a filter in there to find these records... //
+            @Override
-         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            public QQueryFilter getQueryFilter()
-         QProcessMetaData process = instance.getProcess(action.getProcessName());
-         if(process.getInputFields().stream().anyMatch(f -> f.getName().equals(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER)))
             {
                List<Serializable> recordIds = records.stream().map(r -> r.getValueInteger(table.getPrimaryKeyField())).collect(Collectors.toList());
-            QQueryFilter       queryFilter = new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds));
+               return (new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds)));
-            runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, queryFilter);
-         }
-         else
-         {
-            runProcessInput.setRecords(records);
             }
+         });
 
          RunProcessAction runProcessAction = new RunProcessAction();
          RunProcessOutput runProcessOutput = runProcessAction.execute(runProcessInput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizer.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.customizers;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.customizers;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractWidgetRenderer.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.dashboard;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/QuickSightChartRenderer.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.dashboard;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoader.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.dashboard;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QActionInterface.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.interfaces;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java

@@ -140,7 +140,7 @@ public class RunProcessAction
                ///////////////////////
                // Run backend steps //
                ///////////////////////
-               LOG.info("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]");
+               LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]");
                runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState);
             }
             else
@@ -191,7 +191,8 @@ public class RunProcessAction
          if(runProcessInput.getStartAfterStep() == null)
          {
             ///////////////////////////////////////////////////////////////////////////////////
-            // this is fine - it means it's our first time running in the backend.           //
+            // This condition (no state in state-provider, and no start-after-step) means    //
+            // that we're starting a new process!  Init the process state here, then         //
             // Go ahead and store the state that we have (e.g., w/ initial records & values) //
             ///////////////////////////////////////////////////////////////////////////////////
             ProcessState processState = runProcessInput.getProcessState();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java

@@ -72,7 +72,7 @@ public class RecordPipe
 
 
    /*******************************************************************************
-    ** Add a record to the pipe
+    ** Add a record to the pipe.  Will block if the pipe is full.  Will noop if pipe is terminated.
     *******************************************************************************/
    public void addRecord(QRecord record)
    {
@@ -129,7 +129,7 @@ public class RecordPipe
 
 
    /*******************************************************************************
-    ** Add a list of records to the pipe
+    ** Add a list of records to the pipe.  Will block if the pipe is full.  Will noop if pipe is terminated.
     *******************************************************************************/
    public void addRecords(List<QRecord> records)
    {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -291,7 +291,10 @@ public class QInstanceValidator
          {
             if(statusTracking.getType().equals(AutomationStatusTrackingType.FIELD_IN_TABLE))
             {
-               assertCondition(StringUtils.hasContent(statusTracking.getFieldName()), prefix + "statusTracking of type fieldInTable is missing its fieldName");
+               if(assertCondition(StringUtils.hasContent(statusTracking.getFieldName()), prefix + "statusTracking of type fieldInTable is missing its fieldName"))
+               {
+                  assertNoException(() -> table.getField(statusTracking.getFieldName()), prefix + "statusTracking field is not a defined field on this table.");
+               }
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.actions.processes;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java

@@ -371,10 +371,10 @@ public class RunBackendStepInput extends AbstractActionInput
 
 
    /*******************************************************************************
-    ** Getter for a single field's value as a primitive boolean
+    ** Getter for a single field's value as a primitive boolean - with null => false.
     **
     *******************************************************************************/
-   public boolean getValue_boolean(String fieldName)
+   public boolean getValuePrimitiveBoolean(String fieldName)
    {
       Boolean valueAsBoolean = ValueUtils.getValueAsBoolean(getValue(fieldName));
       return (valueAsBoolean != null && valueAsBoolean);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java

@@ -1,8 +1,29 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.actions.processes;
 
 
 /*******************************************************************************
- ** Simple status enum - initially for statusesqqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java in process status lines.
+ ** Simple status enum - initially for statuses in process status lines.
  *******************************************************************************/
 public enum Status
 {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/automation/RecordAutomationInput.java

@@ -1,7 +1,27 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.automation;
 
 
-import java.io.Serializable;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -15,13 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAuto
 public class RecordAutomationInput extends AbstractTableActionInput
 {
    private TableAutomationAction action;
-
-   ////////////////////////////////////////////
-   // todo - why both?  pick one?  or don't? //
-   // maybe - if recordList is null and primaryKeyList isn't, then do the record query in here?
-   ////////////////////////////////////////////
    private List<QRecord> recordList;
-   private List<Serializable> primaryKeyList;
 
 
 
@@ -101,38 +115,4 @@ public class RecordAutomationInput extends AbstractTableActionInput
       return (this);
    }
 
-
-
-   /*******************************************************************************
-    ** Getter for primaryKeyList
-    **
-    *******************************************************************************/
-   public List<Serializable> getPrimaryKeyList()
-   {
-      return primaryKeyList;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for primaryKeyList
-    **
-    *******************************************************************************/
-   public void setPrimaryKeyList(List<Serializable> primaryKeyList)
-   {
-      this.primaryKeyList = primaryKeyList;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for primaryKeyList
-    **
-    *******************************************************************************/
-   public RecordAutomationInput withPrimaryKeyList(List<Serializable> primaryKeyList)
-   {
-      this.primaryKeyList = primaryKeyList;
-      return (this);
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/BarChart.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidget.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QuickSightChart.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java

@@ -90,7 +90,6 @@ public class QRecord implements Serializable
     ** Copy constructor.
     **
     *******************************************************************************/
-   @SuppressWarnings("unchecked")
    public QRecord(QRecord record)
    {
       this.tableName = record.tableName;
@@ -118,8 +117,8 @@ public class QRecord implements Serializable
    /*******************************************************************************
     **
     *******************************************************************************/
-   @SuppressWarnings({ "rawtypes", "unchecked" })
+   @SuppressWarnings({ "unchecked" })
-   private Map doDeepCopy(Map map)
+   private <K, V> Map<K, V> doDeepCopy(Map<K, V> map)
    {
       if(map == null)
       {
@@ -128,10 +127,10 @@ public class QRecord implements Serializable
 
       if(map instanceof Serializable serializableMap)
       {
-         return (Map) SerializationUtils.clone(serializableMap);
+         return (Map<K, V>) SerializationUtils.clone(serializableMap);
       }
 
-      return (new LinkedHashMap(map));
+      return (new LinkedHashMap<>(map));
    }
 
 
@@ -139,8 +138,8 @@ public class QRecord implements Serializable
    /*******************************************************************************
     **
     *******************************************************************************/
-   @SuppressWarnings({ "rawtypes", "unchecked" })
+   @SuppressWarnings({ "unchecked" })
-   private List doDeepCopy(List list)
+   private <T> List<T> doDeepCopy(List<T> list)
    {
       if(list == null)
       {
@@ -149,10 +148,10 @@ public class QRecord implements Serializable
 
       if(list instanceof Serializable serializableList)
       {
-         return (List) SerializationUtils.clone(serializableList);
+         return (List<T>) SerializationUtils.clone(serializableList);
       }
 
-      return (new ArrayList(list));
+      return (new ArrayList<>(list));
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/PollingAutomationProviderMetaData.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.metadata.automation;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.metadata.automation;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PVSValueFormatAndFields.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.metadata.possiblevalues;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTracking.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.metadata.tables.automation;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/AutomationStatusTrackingType.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.metadata.tables.automation;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.metadata.tables.automation;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TableAutomationAction.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.metadata.tables.automation;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TriggerEvent.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.model.metadata.tables.automation;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java

@@ -35,6 +35,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
@@ -43,6 +45,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
  *******************************************************************************/
 public class MockQueryAction implements QueryInterface
 {
+   private static final Logger LOG = LogManager.getLogger(MockQueryAction.class);
+
+
 
    /*******************************************************************************
     **
@@ -68,6 +73,12 @@ public class MockQueryAction implements QueryInterface
             }
 
             queryOutput.addRecord(record);
+
+            if(queryInput.getAsyncJobCallback().wasCancelRequested())
+            {
+               LOG.info("Breaking query job, as requested.");
+               break;
+            }
          }
 
          return (queryOutput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractExtractStep.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
 
 
@@ -19,7 +40,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** should be written to the outputRecordPage.  That is to say, DO NOT use the
  ** recordList in the step input/output objects.
  **
- ** Also - use the transaction member variable - though be aware, it
+ ** Also - use the transaction member variable!!!
  *******************************************************************************/
 public abstract class AbstractLoadStep implements BackendStep
 {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
 
 
@@ -17,7 +38,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** should be written to the outputRecordPage.  That is to say, DO NOT use the
  ** recordList in the step input/output objects.
  *******************************************************************************/
-public abstract class AbstractTransformStep implements BackendStep
+public abstract class AbstractTransformStep implements BackendStep, ProcessSummaryProviderInterface
 {
    private List<QRecord> inputRecordPage  = new ArrayList<>();
    private List<QRecord> outputRecordPage = new ArrayList<>();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java

@@ -1,14 +1,39 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
 
 
+import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
@@ -16,6 +41,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
  *******************************************************************************/
 public class BaseStreamedETLStep
 {
+   private static final Logger LOG = LogManager.getLogger(BaseStreamedETLStep.class);
+
    protected static final int PROCESS_OUTPUT_RECORD_LIST_LIMIT = 20;
 
 
@@ -70,4 +97,20 @@ public class BaseStreamedETLStep
          qPossibleValueTranslator.translatePossibleValuesInRecords(input.getTable(), list);
       }
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected void moveReviewStepAfterValidateStep(RunBackendStepOutput runBackendStepOutput)
+   {
+      LOG.info("Skipping to validation step");
+      ArrayList<String> stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList());
+      LOG.debug("Step list pre: " + stepList);
+      stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW));
+      stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW);
+      runBackendStepOutput.getProcessState().setStepList(stepList);
+      LOG.debug("Step list post: " + stepList);
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
 
 
@@ -12,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 
 
@@ -42,6 +64,7 @@ public class ExtractViaQueryStep extends AbstractExtractStep
       queryInput.setFilter(getQueryFilter(runBackendStepInput));
       queryInput.setRecordPipe(getRecordPipe());
       queryInput.setLimit(getLimit());
+      queryInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
       new QueryAction().execute(queryInput);
 
       ///////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ProcessSummaryProviderInterface.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
 
 
@@ -6,13 +27,13 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine
 
 
 /*******************************************************************************
- **
+ ** Interface for a class that can proivate a ProcessSummary - a list of Process Summary Lines
  *******************************************************************************/
 public interface ProcessSummaryProviderInterface
 {
 
    /*******************************************************************************
-    **
+    ** Note - object needs to be serializable, and List isn't... so, use ArrayList?
     *******************************************************************************/
    ArrayList<ProcessSummaryLine> getProcessSummary(boolean isForResultScreen);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java

@@ -68,6 +68,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
          AbstractLoadStep      loadStep      = getLoadStep(runBackendStepInput);
 
          transformStep.preRun(runBackendStepInput, runBackendStepOutput);
+         loadStep.preRun(runBackendStepInput, runBackendStepOutput);
 
          transaction = loadStep.openTransaction(runBackendStepInput);
          loadStep.setTransaction(transaction);
@@ -86,14 +87,10 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
          updateRecordsWithDisplayValuesAndPossibleValues(runBackendStepInput, loadedRecordList);
          runBackendStepOutput.setRecords(loadedRecordList);
 
-         if(transformStep instanceof ProcessSummaryProviderInterface processSummaryProvider)
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
-         {
+         // get the process summary from the ... transform step?  the load step?  each knows some... todo? //
-            //////////////////////////////////////////////////////////////////////////////////////////////
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
-            // get the process summary from the ... transform step?  the load step?  each knows some... //
+         runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, transformStep.getProcessSummary(true));
-            // TODO!!                                                                                   //
-            //////////////////////////////////////////////////////////////////////////////////////////////
-            runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummaryProvider.getProcessSummary(true));
-         }
 
          transformStep.postRun(runBackendStepInput, runBackendStepOutput);
          loadStep.postRun(runBackendStepInput, runBackendStepOutput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java

@@ -57,11 +57,12 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
       //////////////////////////////////////////////////////////////////////////////////////////////////////////
       // if the do-full-validation flag has already been set, then do the validation step instead of this one //
       //////////////////////////////////////////////////////////////////////////////////////////////////////////
-      boolean supportsFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION);
+      boolean supportsFullValidation = runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION);
-      boolean doFullValidation       = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION);
+      boolean doFullValidation       = runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION);
       if(supportsFullValidation && doFullValidation)
       {
-         skipToValidateStep(runBackendStepOutput);
+         runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
+         moveReviewStepAfterValidateStep(runBackendStepOutput);
          return;
       }
 
@@ -87,7 +88,8 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
       // todo - maybe some future version we do this - maybe based on a user-preference
       // if(supportsFullValidation && recordCount <= limit)
       // {
-      //    skipToValidateStep(runBackendStepOutput);
+      //    runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
+      //    moveReviewStepAfterValidateStep(runBackendStepOutput);
       //    return;
       // }
 
@@ -104,6 +106,7 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
       List<QRecord> previewRecordList = new ArrayList<>();
       new AsyncRecordPipeLoop().run("StreamedETL>Preview>ExtractStep", PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) ->
          {
+            runBackendStepInput.setAsyncJobCallback(status);
             extractStep.run(runBackendStepInput, runBackendStepOutput);
             return (runBackendStepOutput);
          },
@@ -137,22 +140,6 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void skipToValidateStep(RunBackendStepOutput runBackendStepOutput)
-   {
-      LOG.info("Skipping to validation step");
-      runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
-      ArrayList<String> stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList());
-      System.out.println("Step list pre: " + stepList);
-      stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW));
-      stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW);
-      runBackendStepOutput.getProcessState().setStepList(stepList);
-      System.out.println("Step list post: " + stepList);
-   }
-
-
 
    /*******************************************************************************
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java

@@ -56,7 +56,7 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back
       /////////////////////////////////////////////////////////////////////
       // check if we are supported in this process - if not, return noop //
       /////////////////////////////////////////////////////////////////////
-      boolean supportsFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION);
+      boolean supportsFullValidation = runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION);
       if(!supportsFullValidation)
       {
          LOG.info("Process does not support validation, so skipping validation step");
@@ -66,22 +66,17 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back
       ////////////////////////////////////////////////////////////////////////////////
       // check if we've been requested to run in this process - if not, return noop //
       ////////////////////////////////////////////////////////////////////////////////
-      boolean doFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION);
+      boolean doFullValidation = runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION);
       if(!doFullValidation)
       {
          LOG.info("Not requested to do full validation, so skipping validation step");
          return;
       }
 
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // if we're proceeding with full validation, move the review step to be after validation in the step list //
+      // if we're proceeding with full validation, make sure the review step is after validation in the step list //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      ArrayList<String> stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList());
+      moveReviewStepAfterValidateStep(runBackendStepOutput);
-      System.out.println("Step list pre: " + stepList);
-      stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW));
-      stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW);
-      runBackendStepOutput.getProcessState().setStepList(stepList);
-      System.out.println("Step list post: " + stepList);
 
       //////////////////////////////////////////////////////////
       // basically repeat the preview step, but with no limit //
@@ -92,11 +87,6 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back
       extractStep.setRecordPipe(recordPipe);
 
       AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
-      if(!(transformStep instanceof ProcessSummaryProviderInterface processSummaryProvider))
-      {
-         // todo - really?  if this is required, then put it on the AbstractTransformStep class
-         throw (new QException("Transform Step " + transformStep.getClass().getName() + " does not implement ProcessSummaryProviderInterface."));
-      }
       transformStep.preRun(runBackendStepInput, runBackendStepOutput);
 
       List<QRecord> previewRecordList = new ArrayList<>();
@@ -115,7 +105,7 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back
       //////////////////////////////////////////////////////
       // get the process summary from the validation step //
       //////////////////////////////////////////////////////
-      runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY, processSummaryProvider.getProcessSummary(false));
+      runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY, transformStep.getProcessSummary(false));
 
       transformStep.postRun(runBackendStepInput, runBackendStepOutput);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java

@@ -41,10 +41,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
 /*******************************************************************************
  ** Definition for Streamed ETL process that includes a frontend.
  **
- ** This process uses 2 backend steps, and 2 frontend steps, as follows:
+ ** This process uses 3 backend steps, and 2 frontend steps, as follows:
  ** - preview (backend) - does just a little work (limited # of rows), to give the
  **      user a preview of what the final result will be - e.g., some data to seed the review screen
  ** - review (frontend) - a review screen
+ ** - validate (backend) - optionally (per input on review screen), does like the preview step,
+ **      but on all records from the extract step.
  ** - execute (backend) - processes all the rows, does all the work.
  ** - result (frontend) - a result screen
  **
@@ -54,7 +56,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
  ** - Transform - do whatever transformation is needed to the rows.  Done on preview
  **      and execute.  Always works with a "page" of records at a time.
  ** - Load - store the records into the backend, as appropriate.  Always works
- **      with a "page" of records at a time.
+ **      with a "page" of records at a time.  Only called by execute step.
  *******************************************************************************/
 public class StreamedETLWithFrontendProcess
 {

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationExecutorTest.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.automation.polling;
 
 
@@ -10,11 +31,8 @@ import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
-import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -76,19 +94,16 @@ class PollingAutomationExecutorTest
       /////////////////////////////////////////////////
       // query for the records - assert their status //
       /////////////////////////////////////////////////
-      QueryInput queryInput = new QueryInput(qInstance);
+      List<QRecord> records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
-      queryInput.setSession(new QSession());
+      assertEquals(2, records.size());
-      queryInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
-      QueryOutput queryOutput = new QueryAction().execute(queryInput);
-      assertEquals(2, queryOutput.getRecords().size());
 
-      Optional<QRecord> optionalPerson1 = queryOutput.getRecords().stream().filter(r -> r.getValueInteger("id") == 1).findFirst();
+      Optional<QRecord> optionalPerson1 = records.stream().filter(r -> r.getValueInteger("id") == 1).findFirst();
       assertThat(optionalPerson1).isPresent();
       QRecord person1 = optionalPerson1.get();
       assertThat(person1.getValueString("firstName")).isEqualTo("John");
       assertThat(person1.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName())).isEqualTo(AutomationStatus.OK.getId());
 
-      Optional<QRecord> optionalPerson2 = queryOutput.getRecords().stream().filter(r -> r.getValueInteger("id") == 2).findFirst();
+      Optional<QRecord> optionalPerson2 = records.stream().filter(r -> r.getValueInteger("id") == 2).findFirst();
       assertThat(optionalPerson2).isPresent();
       QRecord person2 = optionalPerson2.get();
       assertThat(person2.getValueString("firstName")).isEqualTo("Jim" + TestUtils.CheckAge.SUFFIX_FOR_MINORS);

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationRunnerTest.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.automation.polling;
 
 
@@ -14,8 +35,20 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcessTest;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
@@ -51,15 +84,15 @@ class PollingAutomationRunnerTest
       QInstance               qInstance               = TestUtils.defineInstance();
       PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, TestUtils.POLLING_AUTOMATION, null);
 
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // insert 2 person records, one who should be both updated by the insert action, and should be logged by logger-on-update automation //
+      // insert 2 person records, both updated by the insert action, and 1 logged by logger-on-update automation //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
       InsertInput insertInput = new InsertInput(qInstance);
       insertInput.setSession(new QSession());
       insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
       insertInput.setRecords(List.of(
          new QRecord().withValue("id", 1).withValue("firstName", "Tim").withValue("birthDate", LocalDate.now()),
-         new QRecord().withValue("id", 2).withValue("firstName", "Darin")
+         new QRecord().withValue("id", 2).withValue("firstName", "Darin").withValue("birthDate", LocalDate.now())
       ));
       new InsertAction().execute(insertInput);
       assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS);
@@ -72,15 +105,11 @@ class PollingAutomationRunnerTest
       assertThat(TestUtils.LogPersonUpdate.updatedIds).isNullOrEmpty();
       assertAllRecordsAutomationStatus(AutomationStatus.OK);
 
-      ////////////////////////////////////////////
+      /////////////////////////////////////////
-      // make sure the minor person was updated //
+      // make sure both persons were updated //
-      ////////////////////////////////////////////
+      /////////////////////////////////////////
-      Optional<QRecord> updatedMinorRecord = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY).stream().filter(r -> r.getValueInteger("id").equals(1)).findFirst();
+      assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY))
-      assertThat(updatedMinorRecord)
+         .allMatch(r -> r.getValueString("firstName").endsWith(TestUtils.CheckAge.SUFFIX_FOR_MINORS));
-         .isPresent()
-         .get()
-         .extracting(r -> r.getValueString("firstName"))
-         .isEqualTo("Tim" + TestUtils.CheckAge.SUFFIX_FOR_MINORS);
 
       /////////////////////////////////////////////////////////////////////////////////////////
       // run automations again - make sure that there haven't been any updates triggered yet //
@@ -179,7 +208,7 @@ class PollingAutomationRunnerTest
 
 
    /*******************************************************************************
-    ** Test a cycle that does an insert, some automations, then and an update, and more automations.
+    ** Test running a process for automation, instead of a code ref.
     *******************************************************************************/
    @Test
    void testRunningProcess() throws QException
@@ -187,9 +216,9 @@ class PollingAutomationRunnerTest
       QInstance               qInstance               = TestUtils.defineInstance();
       PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(qInstance, TestUtils.POLLING_AUTOMATION, null);
 
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      ////////////////////////////////////////////////////////////////////
-      // insert 2 person records, one who should be both updated by the insert action, and should be logged by logger-on-update automation //
+      // insert 2 person records, 1 to trigger the "increaseAge" action //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      ////////////////////////////////////////////////////////////////////
       InsertInput insertInput = new InsertInput(qInstance);
       insertInput.setSession(new QSession());
       insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
@@ -214,6 +243,62 @@ class PollingAutomationRunnerTest
    }
 
 
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testRunningEtlWithFrontendProcess() throws QException
+   {
+      QInstance instance = TestUtils.defineInstance();
+
+      ////////////////////////////////////////////////////////
+      // define the process - an ELT from Shapes to Persons //
+      ////////////////////////////////////////////////////////
+      QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
+         TestUtils.TABLE_NAME_SHAPE,
+         TestUtils.TABLE_NAME_PERSON,
+         ExtractViaQueryStep.class,
+         StreamedETLWithFrontendProcessTest.TestTransformShapeToPersonStep.class,
+         LoadViaInsertStep.class);
+      process.setName("shapeToPersonETLProcess");
+      process.setTableName(TestUtils.TABLE_NAME_SHAPE);
+      instance.addProcess(process);
+
+      ///////////////////////////////////////////////////////
+      // switch the person table to use the memory backend //
+      ///////////////////////////////////////////////////////
+      instance.getTable(TestUtils.TABLE_NAME_PERSON).setBackendName(TestUtils.MEMORY_BACKEND_NAME);
+
+      ///////////////////////////////////////////////////////////////////////
+      // add a post-insert process to the shape table, to run this ELT job //
+      ///////////////////////////////////////////////////////////////////////
+      instance.getTable(TestUtils.TABLE_NAME_SHAPE)
+         .withField(new QFieldMetaData("automationStatus", QFieldType.INTEGER))
+         .setAutomationDetails(new QTableAutomationDetails()
+         .withProviderName(TestUtils.POLLING_AUTOMATION)
+         .withStatusTracking(new AutomationStatusTracking().withType(AutomationStatusTrackingType.FIELD_IN_TABLE).withFieldName("automationStatus"))
+         .withAction(new TableAutomationAction()
+            .withName("shapeToPerson")
+            .withTriggerEvent(TriggerEvent.POST_INSERT)
+            .withProcessName("shapeToPersonETLProcess")
+         )
+      );
+
+      TestUtils.insertDefaultShapes(instance);
+
+      PollingAutomationRunner pollingAutomationRunner = new PollingAutomationRunner(instance, TestUtils.POLLING_AUTOMATION, null);
+      pollingAutomationRunner.run();
+
+      List<QRecord> postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_PERSON);
+      assertThat(postList)
+         .as("Should have inserted Circle").anyMatch(qr -> qr.getValue("lastName").equals("Circle"))
+         .as("Should have inserted Triangle").anyMatch(qr -> qr.getValue("lastName").equals("Triangle"))
+         .as("Should have inserted Square").anyMatch(qr -> qr.getValue("lastName").equals("Square"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/PersonsByCreateDateBarChart.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.dashboard;
 
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/WidgetDataLoaderTest.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.actions.dashboard;
 
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java

@@ -372,9 +372,9 @@ public class RunProcessTest
          ))
       );
 
-      /////////////////////////////////////////////////////////////////////////////
+      ////////////////////////////////////////////////////////////////////////////////
-      // make sure that if we run by default, we get stop on both frontend steps //
+      // make sure that if we run by default, we get stopped on both frontend steps //
-      /////////////////////////////////////////////////////////////////////////////
+      ////////////////////////////////////////////////////////////////////////////////
       RunProcessInput request = new RunProcessInput(qInstance);
       request.setSession(TestUtils.getMockSession());
       request.setProcessName(processName);

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -721,9 +721,8 @@ class QInstanceValidatorTest
       assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().getStatusTracking().setFieldName(""),
          "missing its fieldName");
 
-      //////////////////////////////////////////////////
+      assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getAutomationDetails().getStatusTracking().setFieldName("notARealField"),
-      // todo - make sure it's a field in the table?? //
+         "not a defined field");
-      //////////////////////////////////////////////////
    }
 
 
@@ -791,14 +790,6 @@ class QInstanceValidatorTest
          },
          "unrecognized processName");
 
-      assertValidationSuccess((qInstance) ->
-      {
-         qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
-         TableAutomationAction action = getAction0(qInstance);
-         action.setCodeReference(null);
-         action.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE);
-      });
-
       assertValidationFailureReasons((qInstance) ->
          {
             TableAutomationAction action = getAction0(qInstance);
@@ -826,9 +817,9 @@ class QInstanceValidatorTest
 
       assertValidationFailureReasons((qInstance) ->
          {
-            qInstance.getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE).setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
             TableAutomationAction action = getAction0(qInstance);
-            action.setProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE);
+            action.setCodeReference(new QCodeReference(TestUtils.CheckAge.class));
+            action.setProcessName(TestUtils.PROCESS_NAME_INCREASE_BIRTHDATE);
          },
          "has both");
    }
@@ -857,14 +848,6 @@ class QInstanceValidatorTest
             );
          },
          "unrecognized field");
-
-      assertValidationSuccess((qInstance) ->
-         {
-            TableAutomationAction action = getAction0(qInstance);
-            action.setFilter(new QQueryFilter()
-               .withCriteria(new QFilterCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, List.of(1701)))
-            );
-         });
    }
 
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
 
 
@@ -36,7 +57,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 /*******************************************************************************
  ** Unit test for StreamedETLWithFrontendProcess
  *******************************************************************************/
-class StreamedETLWithFrontendProcessTest
+public class StreamedETLWithFrontendProcessTest
 {
 
    /*******************************************************************************
@@ -64,7 +85,7 @@ class StreamedETLWithFrontendProcessTest
       ////////////////////////////////////////////////////////
       QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
          TestUtils.TABLE_NAME_SHAPE,
-         TestUtils.TABLE_NAME_PERSON,
+         TestUtils.TABLE_NAME_PERSON_MEMORY,
          ExtractViaQueryStep.class,
          TestTransformShapeToPersonStep.class,
          LoadViaInsertStep.class);
@@ -72,11 +93,6 @@ class StreamedETLWithFrontendProcessTest
       process.setTableName(TestUtils.TABLE_NAME_SHAPE);
       instance.addProcess(process);
 
-      ///////////////////////////////////////////////////////
-      // switch the person table to use the memory backend //
-      ///////////////////////////////////////////////////////
-      instance.getTable(TestUtils.TABLE_NAME_PERSON).setBackendName(TestUtils.MEMORY_BACKEND_NAME);
-
       TestUtils.insertDefaultShapes(instance);
 
       /////////////////////
@@ -84,7 +100,7 @@ class StreamedETLWithFrontendProcessTest
       /////////////////////
       runProcess(instance, process);
 
-      List<QRecord> postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_PERSON);
+      List<QRecord> postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_PERSON_MEMORY);
       assertThat(postList)
          .as("Should have inserted Circle").anyMatch(qr -> qr.getValue("lastName").equals("Circle"))
          .as("Should have inserted Triangle").anyMatch(qr -> qr.getValue("lastName").equals("Triangle"))
@@ -277,7 +293,7 @@ class StreamedETLWithFrontendProcessTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private RunProcessOutput runProcess(QInstance instance, QProcessMetaData process) throws QException
+   public RunProcessOutput runProcess(QInstance instance, QProcessMetaData process) throws QException
    {
       return (runProcess(instance, process, new HashMap<>(), new Callback()));
    }
@@ -338,6 +354,14 @@ class StreamedETLWithFrontendProcessTest
             getOutputRecordPage().add(newQRecord);
          }
       }
+
+
+
+      @Override
+      public ArrayList<ProcessSummaryLine> getProcessSummary(boolean isForResultScreen)
+      {
+         return null;
+      }
    }
 
 
@@ -420,6 +444,14 @@ class StreamedETLWithFrontendProcessTest
             getOutputRecordPage().add(updatedQRecord);
          }
       }
+
+
+
+      @Override
+      public ArrayList<ProcessSummaryLine> getProcessSummary(boolean isForResultScreen)
+      {
+         return null;
+      }
    }
 
 

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/FilesystemTableCustomizers.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.module.filesystem.base.actions;
 
 

qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/PersonsByCreateDateBarChart.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.javalin;
 
 

qqq-sample-project/src/main/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChart.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.sampleapp.dashboard.widgets;
 
 

qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.sampleapp.processes.clonepeople;
 
 

qqq-sample-project/src/test/java/com/kingsrook/sampleapp/SampleJavalinServerTest.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.sampleapp;
 
 

qqq-sample-project/src/test/java/com/kingsrook/sampleapp/dashboard/widgets/PersonsByCreateDateBarChartTest.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.sampleapp.dashboard.widgets;
 
 

qqq-sample-project/src/test/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStepTest.java

@@ -1,3 +1,24 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.sampleapp.processes.clonepeople;