Merged dev into feature/quartz-scheduler

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.instances;
 
 import java.io.Serializable;
 import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.text.ParseException;
 import java.util.ArrayList;
@@ -46,6 +47,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface;
 import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -55,9 +57,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QSupplementalInstanceMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.ParentWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@@ -67,16 +71,21 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QSupplementalProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
@@ -93,6 +102,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda;
@@ -115,6 +125,8 @@ public class QInstanceValidator
 
    private boolean printWarnings = false;
 
+   private static ListingHash<Class<?>, QInstanceValidatorPluginInterface<?>> validatorPlugins = new ListingHash<>();
+
    private List<String> errors = new ArrayList<>();
 
 
@@ -175,6 +187,8 @@ public class QInstanceValidator
          validateSupplementalMetaData(qInstance);
 
          validateUniqueTopLevelNames(qInstance);
+
+         runPlugins(QInstance.class, qInstance, qInstance);
       }
       catch(Exception e)
       {
@@ -193,6 +207,57 @@ public class QInstanceValidator
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void addValidatorPlugin(QInstanceValidatorPluginInterface<?> plugin)
+   {
+      Optional<Method> validateMethod = Arrays.stream(plugin.getClass().getDeclaredMethods())
+         .filter(m -> m.getName().equals("validate")
+            && m.getParameterCount() == 3
+            && !m.getParameterTypes()[0].equals(Object.class)
+            && m.getParameterTypes()[1].equals(QInstance.class)
+            && m.getParameterTypes()[2].equals(QInstanceValidator.class)
+         ).findFirst();
+
+      if(validateMethod.isPresent())
+      {
+         Class<?> parameterType = validateMethod.get().getParameterTypes()[0];
+         validatorPlugins.add(parameterType, plugin);
+      }
+      else
+      {
+         LOG.warn("Could not find validate method on validator plugin [" + plugin.getClass().getName() + "] (to infer type being validated) - this plugin will not be used.");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void removeAllValidatorPlugins()
+   {
+      validatorPlugins.clear();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private <T> void runPlugins(Class<T> c, T t, QInstance qInstance)
+   {
+      for(QInstanceValidatorPluginInterface<?> plugin : CollectionUtils.nonNullList(validatorPlugins.get(c)))
+      {
+         @SuppressWarnings("unchecked")
+         QInstanceValidatorPluginInterface<T> processPlugin = (QInstanceValidatorPluginInterface<T>) plugin;
+         processPlugin.validate(t, qInstance, this);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -201,6 +266,8 @@ public class QInstanceValidator
       for(QSupplementalInstanceMetaData supplementalInstanceMetaData : CollectionUtils.nonNullMap(qInstance.getSupplementalMetaData()).values())
       {
          supplementalInstanceMetaData.validate(qInstance, this);
+
+         runPlugins(QSupplementalInstanceMetaData.class, supplementalInstanceMetaData, qInstance);
       }
    }
 
@@ -238,6 +305,8 @@ public class QInstanceValidator
             {
                assertCondition(qInstance.getPossibleValueSource(securityKeyType.getPossibleValueSourceName()) != null, "Unrecognized possibleValueSourceName in securityKeyType: " + name);
             }
+
+            runPlugins(QSecurityKeyType.class, securityKeyType, qInstance);
          }
       });
    }
@@ -284,6 +353,8 @@ public class QInstanceValidator
                assertNoException(() -> qInstance.getTable(join.getRightTable()).getField(orderBy.getFieldName()), "Field name " + orderBy.getFieldName() + " in orderBy for join " + joinName + " is not a defined field in the right-table " + join.getRightTable());
             }
          }
+
+         runPlugins(QJoinMetaData.class, join, qInstance);
       });
    }
 
@@ -352,6 +423,8 @@ public class QInstanceValidator
                   validateScheduleMetaData(sqsQueueProvider.getSchedule(), qInstance, "SQSQueueProvider " + name + ", schedule: ");
                }
             }
+
+            runPlugins(QQueueProviderMetaData.class, queueProvider, qInstance);
          });
       }
 
@@ -366,6 +439,8 @@ public class QInstanceValidator
             {
                assertCondition(qInstance.getProcesses() != null && qInstance.getProcess(queue.getProcessName()) != null, "Unrecognized processName for queue: " + name);
             }
+
+            runPlugins(QQueueMetaData.class, queue, qInstance);
          });
       }
    }
@@ -384,6 +459,8 @@ public class QInstanceValidator
             assertCondition(Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + ".");
 
             backend.performValidation(this);
+
+            runPlugins(QBackendMetaData.class, backend, qInstance);
          });
       }
    }
@@ -406,6 +483,8 @@ public class QInstanceValidator
             {
                validateScheduleMetaData(automationProvider.getSchedule(), qInstance, "automationProvider " + name + ", schedule: ");
             }
+
+            runPlugins(QAutomationProviderMetaData.class, automationProvider, qInstance);
          });
       }
    }
@@ -424,6 +503,8 @@ public class QInstanceValidator
          {
             validateSimpleCodeReference("Instance Authentication meta data customizer ", authentication.getCustomizer(), QAuthenticationModuleCustomizerInterface.class);
          }
+
+         runPlugins(QAuthenticationMetaData.class, authentication, qInstance);
       }
    }
 
@@ -546,6 +627,8 @@ public class QInstanceValidator
             {
                supplementalTableMetaData.validate(qInstance, table, this);
             }
+
+            runPlugins(QTableMetaData.class, table, qInstance);
          });
       }
    }
@@ -1344,6 +1427,7 @@ public class QInstanceValidator
                supplementalProcessMetaData.validate(qInstance, process, this);
             }
 
+            runPlugins(QProcessMetaData.class, process, qInstance);
          });
       }
    }
@@ -1494,6 +1578,8 @@ public class QInstanceValidator
                   // view.getTitleFormat(); view.getTitleFields(); // validate these match?
                }
             }
+
+            runPlugins(QReportMetaData.class, report, qInstance);
          });
       }
    }
@@ -1614,9 +1700,9 @@ public class QInstanceValidator
                }
             }
 
-            //////////////////////////////////////////
-            // validate field sections in the table //
-            //////////////////////////////////////////
+            ////////////////////////////////////////
+            // validate field sections in the app //
+            ////////////////////////////////////////
             Set<String> childNamesInSections = new HashSet<>();
             if(app.getSections() != null)
             {
@@ -1644,6 +1730,8 @@ public class QInstanceValidator
             {
                assertCondition(qInstance.getWidget(widgetName) != null, "App " + appName + " widget " + widgetName + " is not a recognized widget.");
             }
+
+            runPlugins(QAppMetaData.class, app, qInstance);
          });
       }
    }
@@ -1676,6 +1764,8 @@ public class QInstanceValidator
                      }
                   }
                }
+
+               runPlugins(QWidgetMetaDataInterface.class, widget, qInstance);
             }
          );
       }
@@ -1761,6 +1851,8 @@ public class QInstanceValidator
                   }
                   default -> errors.add("Unexpected possibleValueSource type: " + possibleValueSource.getType());
                }
+
+               runPlugins(QPossibleValueSource.class, possibleValueSource, qInstance);
             }
          });
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/BasepullExtractStepValidator.java

@@ -0,0 +1,87 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.validation.plugins;
+
+
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullExtractStepInterface;
+
+
+/*******************************************************************************
+ ** instance validator plugin, to ensure that a process which is a basepull uses
+ ** an extract step marked for basepulls.
+ *******************************************************************************/
+public class BasepullExtractStepValidator implements QInstanceValidatorPluginInterface<QProcessMetaData>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void validate(QProcessMetaData process, QInstance qInstance, QInstanceValidator qInstanceValidator)
+   {
+      ///////////////////////////////////////////////////////////////////////////
+      // if there's no basepull config on the process, don't do any validation //
+      ///////////////////////////////////////////////////////////////////////////
+      if(process.getBasepullConfiguration() == null)
+      {
+         return;
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // try to find an input field in the process, w/ a defaultValue that's a QCodeReference //
+      // and is an instance of BasepullExtractStepInterface                                   //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      boolean foundBasepullExtractStep = false;
+      for(QFieldMetaData field : process.getInputFields())
+      {
+         if(field.getDefaultValue() != null && field.getDefaultValue() instanceof QCodeReference codeReference)
+         {
+            try
+            {
+               BasepullExtractStepInterface extractStep = QCodeLoader.getAdHoc(BasepullExtractStepInterface.class, codeReference);
+               if(extractStep != null)
+               {
+                  foundBasepullExtractStep = true;
+               }
+            }
+            catch(Exception e)
+            {
+               //////////////////////////////////////////////////////
+               // ok, just means we haven't found our extract step //
+               //////////////////////////////////////////////////////
+            }
+         }
+      }
+
+      ///////////////////////////////////////////////////////////
+      // validate we could find a BasepullExtractStepInterface //
+      ///////////////////////////////////////////////////////////
+      qInstanceValidator.assertCondition(foundBasepullExtractStep, "Process [" + process.getName() + "] has a basepullConfiguration, but does not have a field with a default value that is a BasepullExtractStepInterface CodeReference");
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/QInstanceValidatorPluginInterface.java

@@ -0,0 +1,41 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.validation.plugins;
+
+
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+
+
+/*******************************************************************************
+ ** Interface for additional / optional q instance validators.  Some will be
+ ** provided by QQQ - others can be defined by applications.
+ *******************************************************************************/
+public interface QInstanceValidatorPluginInterface<T>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void validate(T object, QInstance qInstance, QInstanceValidator qInstanceValidator);
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java

@@ -0,0 +1,153 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.logging;
+
+
+import org.apache.logging.log4j.Level;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+
+/*******************************************************************************
+ ** A log message, which can be "collected" by the QCollectingLogger.
+ *******************************************************************************/
+public class CollectedLogMessage
+{
+   private Level     level;
+   private String    message;
+   private Throwable exception;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public CollectedLogMessage()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for message
+    *******************************************************************************/
+   public String getMessage()
+   {
+      return (this.message);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for message
+    *******************************************************************************/
+   public void setMessage(String message)
+   {
+      this.message = message;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for message
+    *******************************************************************************/
+   public CollectedLogMessage withMessage(String message)
+   {
+      this.message = message;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for exception
+    *******************************************************************************/
+   public Throwable getException()
+   {
+      return (this.exception);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for exception
+    *******************************************************************************/
+   public void setException(Throwable exception)
+   {
+      this.exception = exception;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for exception
+    *******************************************************************************/
+   public CollectedLogMessage withException(Throwable exception)
+   {
+      this.exception = exception;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for level
+    **
+    *******************************************************************************/
+   public Level getLevel()
+   {
+      return level;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for level
+    **
+    *******************************************************************************/
+   public void setLevel(Level level)
+   {
+      this.level = level;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for level
+    **
+    *******************************************************************************/
+   public CollectedLogMessage withLevel(Level level)
+   {
+      this.level = level;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public JSONObject getMessageAsJSONObject() throws JSONException
+   {
+      return (new JSONObject(getMessage()));
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QCollectingLogger.java

@@ -0,0 +1,155 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.logging;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.message.Message;
+import org.apache.logging.log4j.message.ObjectMessage;
+import org.apache.logging.log4j.message.SimpleMessageFactory;
+import org.apache.logging.log4j.simple.SimpleLogger;
+import org.apache.logging.log4j.util.PropertiesUtil;
+
+
+/*******************************************************************************
+ ** QQQ log4j implementation, used within a QLogger, to "collect" log messages
+ ** in an internal list - the idea being - for tests, to assert that logs happened.
+ *******************************************************************************/
+public class QCollectingLogger extends SimpleLogger
+{
+   private List<CollectedLogMessage> collectedMessages = new ArrayList<>();
+
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+   // just in case one of these gets activated, and left on, put a limit on how many messages we'll collect //
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+   private int capacity = 100;
+
+   private Logger logger;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public QCollectingLogger(Logger logger)
+   {
+      super(logger.getName(), logger.getLevel(), false, false, true, false, "", new SimpleMessageFactory(), new PropertiesUtil(new Properties()), System.out);
+      this.logger = logger;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void logMessage(String fqcn, Level level, Marker marker, Message message, Throwable throwable)
+   {
+      ////////////////////////////////////////////
+      // add this log message to our collection //
+      ////////////////////////////////////////////
+      collectedMessages.add(new CollectedLogMessage()
+         .withLevel(level)
+         .withMessage(message.getFormattedMessage())
+         .withException(throwable));
+
+      ////////////////////////////////////////////////////////////////////////////////////////
+      // if we've gone over our capacity, remove the 1st entry until we're back at capacity //
+      ////////////////////////////////////////////////////////////////////////////////////////
+      while(collectedMessages.size() > capacity)
+      {
+         collectedMessages.remove(0);
+      }
+
+      //////////////////////////////////////////////////////////////////////
+      // update the message that we log to indicate that we collected it. //
+      // if it looks like JSON, insert as a name:value pair; else text.   //
+      //////////////////////////////////////////////////////////////////////
+      String formattedMessage = message.getFormattedMessage();
+      String updatedMessage;
+      if(formattedMessage.startsWith("{"))
+      {
+         updatedMessage = """
+            {"collected":true,""" + formattedMessage.substring(1);
+      }
+      else
+      {
+         updatedMessage = "[Collected] " + formattedMessage;
+      }
+      ObjectMessage myMessage = new ObjectMessage(updatedMessage);
+
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // log the message with the original log4j logger, with our slightly updated message //
+      ///////////////////////////////////////////////////////////////////////////////////////
+      logger.logMessage(level, marker, fqcn, null, myMessage, throwable);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for logger
+    **
+    *******************************************************************************/
+   public void setLogger(Logger logger)
+   {
+      this.logger = logger;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for collectedMessages
+    **
+    *******************************************************************************/
+   public List<CollectedLogMessage> getCollectedMessages()
+   {
+      return collectedMessages;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void clear()
+   {
+      this.collectedMessages.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for capacity
+    **
+    *******************************************************************************/
+   public void setCapacity(int capacity)
+   {
+      this.capacity = capacity;
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java

@@ -119,6 +119,34 @@ public class QLogger
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QCollectingLogger activateCollectingLoggerForClass(Class<?> c)
+   {
+      Logger            loggerFromLogManager = LogManager.getLogger(c);
+      QCollectingLogger collectingLogger     = new QCollectingLogger(loggerFromLogManager);
+
+      QLogger qLogger = getLogger(c);
+      qLogger.setLogger(collectingLogger);
+
+      return collectingLogger;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void deactivateCollectingLoggerForClass(Class<?> c)
+   {
+      Logger  loggerFromLogManager = LogManager.getLogger(c);
+      QLogger qLogger              = getLogger(c);
+      qLogger.setLogger(loggerFromLogManager);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -518,7 +546,7 @@ public class QLogger
    /*******************************************************************************
     **
     *******************************************************************************/
-   private String makeJsonString(String message, Throwable t, List<LogPair> logPairList)
+   protected String makeJsonString(String message, Throwable t, List<LogPair> logPairList)
    {
       if(logPairList == null)
       {
@@ -620,4 +648,15 @@ public class QLogger
       exceptionList.get(0).setHasLoggedLevel(level);
       return (level);
    }
+
+
+
+   /*******************************************************************************
+    ** Setter for logger
+    **
+    *******************************************************************************/
+   private void setLogger(Logger logger)
+   {
+      this.logger = logger;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java

@@ -287,16 +287,22 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
             //////////////////////////////////////////////////////////////////////////////////////
             if(!recordsToUpdate.isEmpty() && !isReview)
             {
-               LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("count", recordsToUpdate.size()));
+               LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("countByStatus", countByStatus));
                new UpdateAction().execute(new UpdateInput(tableName).withRecords(recordsToUpdate).withOmitTriggeringAutomations(true));
             }
 
-            for(Map.Entry<String, Integer> entry : countByStatus.entrySet())
+            ///////////////////////////////////////////////////
+            // on the review step, add records to the output //
+            ///////////////////////////////////////////////////
+            if(isReview)
             {
-               runBackendStepOutput.addRecord(new QRecord()
-                  .withValue("tableName", QContext.getQInstance().getTable(tableName).getLabel())
-                  .withValue("badStatus", entry.getKey())
-                  .withValue("count", entry.getValue()));
+               for(Map.Entry<String, Integer> entry : countByStatus.entrySet())
+               {
+                  runBackendStepOutput.addRecord(new QRecord()
+                     .withValue("tableName", QContext.getQInstance().getTable(tableName).getLabel())
+                     .withValue("badStatus", entry.getKey())
+                     .withValue("count", entry.getValue()));
+               }
             }
 
             return (recordsToUpdate.size());

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/BasepullExtractStepInterface.java

@@ -0,0 +1,30 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
+
+
+/*******************************************************************************
+ ** marker - used to indicate that a step is meant to be used  for basepull extracts.
+ *******************************************************************************/
+public interface BasepullExtractStepInterface
+{
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java

@@ -38,7 +38,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith
 /*******************************************************************************
  ** Version of ExtractViaQueryStep that knows how to set up a basepull query.
  *******************************************************************************/
-public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
+public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep implements BasepullExtractStepInterface
 {
 
    /*******************************************************************************