Merged dev into feature/quartz-scheduler

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.instances.validation.plugins.AlwaysFailsProcessValidatorPlugin;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@@ -99,7 +100,7 @@ import static org.junit.jupiter.api.Assertions.fail;
  ** Unit test for QInstanceValidator.
  **
  *******************************************************************************/
-class QInstanceValidatorTest extends BaseTest
+public class QInstanceValidatorTest extends BaseTest
 {
 
    /*******************************************************************************
@@ -490,6 +491,35 @@ class QInstanceValidatorTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test_validatorPlugins()
+   {
+      try
+      {
+         QInstanceValidator.addValidatorPlugin(new AlwaysFailsProcessValidatorPlugin());
+
+         ////////////////////////////////////////
+         // make sure our always failer fails. //
+         ////////////////////////////////////////
+         assertValidationFailureReasonsAllowingExtraReasons((qInstance) -> {
+         }, "always fail");
+      }
+      finally
+      {
+         QInstanceValidator.removeAllValidatorPlugins();
+
+         ////////////////////////////////////////////////////
+         // make sure if remove all plugins, we don't fail //
+         ////////////////////////////////////////////////////
+         assertValidationSuccess((qInstance) -> {});
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Test that a table with no fields fails.
     **
@@ -2072,9 +2102,9 @@ class QInstanceValidatorTest extends BaseTest
     ** failed validation with reasons that match the supplied vararg-reasons (but allow
     ** more reasons - e.g., helpful when one thing we're testing causes other errors).
     *******************************************************************************/
-   private void assertValidationFailureReasonsAllowingExtraReasons(Consumer<QInstance> setup, String... reasons)
+   public static void assertValidationFailureReasonsAllowingExtraReasons(Consumer<QInstance> setup, String... expectedReasons)
    {
-      assertValidationFailureReasons(setup, true, reasons);
+      assertValidationFailureReasons(setup, true, expectedReasons);
    }
 
 
@@ -2084,9 +2114,9 @@ class QInstanceValidatorTest extends BaseTest
     ** failed validation with reasons that match the supplied vararg-reasons (and
     ** require that exact # of reasons).
     *******************************************************************************/
-   private void assertValidationFailureReasons(Consumer<QInstance> setup, String... reasons)
+   public static void assertValidationFailureReasons(Consumer<QInstance> setup, String... expectedReasons)
    {
-      assertValidationFailureReasons(setup, false, reasons);
+      assertValidationFailureReasons(setup, false, expectedReasons);
    }
 
 
@@ -2094,7 +2124,7 @@ class QInstanceValidatorTest extends BaseTest
    /*******************************************************************************
     ** Implementation for the overloads of this name.
     *******************************************************************************/
-   private void assertValidationFailureReasons(Consumer<QInstance> setup, boolean allowExtraReasons, String... reasons)
+   public static void assertValidationFailureReasons(Consumer<QInstance> setup, boolean allowExtraReasons, String... expectedReasons)
    {
       try
       {
@@ -2105,17 +2135,27 @@ class QInstanceValidatorTest extends BaseTest
       }
       catch(QInstanceValidationException e)
       {
-         if(!allowExtraReasons)
-         {
-            int noOfReasons = e.getReasons() == null ? 0 : e.getReasons().size();
-            assertEquals(reasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", reasons)
-               + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", e.getReasons()) : "--"));
-         }
+         assertValidationFailureReasons(allowExtraReasons, e.getReasons(), expectedReasons);
+      }
+   }
 
-         for(String reason : reasons)
-         {
-            assertReason(reason, e);
-         }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void assertValidationFailureReasons(boolean allowExtraReasons, List<String> actualReasons, String... expectedReasons)
+   {
+      if(!allowExtraReasons)
+      {
+         int noOfReasons = actualReasons == null ? 0 : actualReasons.size();
+         assertEquals(expectedReasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", expectedReasons)
+            + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--"));
+      }
+
+      for(String reason : expectedReasons)
+      {
+         assertReason(reason, actualReasons);
       }
    }
 
@@ -2145,11 +2185,11 @@ class QInstanceValidatorTest extends BaseTest
     ** the list of reasons in the QInstanceValidationException.
     **
     *******************************************************************************/
-   private void assertReason(String reason, QInstanceValidationException e)
+   public static void assertReason(String reason, List<String> actualReasons)
    {
-      assertNotNull(e.getReasons(), "Expected there to be a reason for the failure (but there was not)");
-      assertThat(e.getReasons())
-         .withFailMessage("Expected any of:\n%s\nTo match: [%s]", e.getReasons(), reason)
+      assertNotNull(actualReasons, "Expected there to be a reason for the failure (but there was not)");
+      assertThat(actualReasons)
+         .withFailMessage("Expected any of:\n%s\nTo match: [%s]", actualReasons, reason)
          .anyMatch(s -> s.contains(reason));
    }
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/AlwaysFailsProcessValidatorPlugin.java

@@ -0,0 +1,45 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.validation.plugins;
+
+
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+
+
+/*******************************************************************************
+ ** test instance of a validator plugin, that always adds an error
+ *******************************************************************************/
+public class AlwaysFailsProcessValidatorPlugin implements QInstanceValidatorPluginInterface<QProcessMetaData>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void validate(QProcessMetaData object, QInstance qInstance, QInstanceValidator qInstanceValidator)
+   {
+      qInstanceValidator.getErrors().add("I always fail.");
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/BasepullExtractStepValidatorTest.java

@@ -0,0 +1,109 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.validation.plugins;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
+import com.kingsrook.qqq.backend.core.processes.implementations.basepull.ExtractViaBasepullQueryStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcessTest;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qqq.backend.core.instances.QInstanceValidatorTest.assertValidationFailureReasons;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Unit test for BasepullExtractStepValidator 
+ *******************************************************************************/
+class BasepullExtractStepValidatorTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testNoExtractStepAtAllFails()
+   {
+      QInstance          qInstance = QContext.getQInstance();
+      QInstanceValidator validator = new QInstanceValidator();
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // turns out, our "basepullTestProcess" doesn't have an extract step, so that makes this condition fire. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+      new BasepullExtractStepValidator().validate(qInstance.getProcess(TestUtils.PROCESS_NAME_BASEPULL), qInstance, validator);
+      assertValidationFailureReasons(false, validator.getErrors(), "does not have a field with a default value that is a BasepullExtractStepInterface CodeReference");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testExtractViaQueryNotBasePull()
+   {
+      QInstance          qInstance = QContext.getQInstance();
+      QInstanceValidator validator = new QInstanceValidator();
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // set up a streamed-etl process, but with an ExtractViaQueryStep instead of a basepull - it should fail! //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      new BasepullExtractStepValidator().validate(StreamedETLWithFrontendProcess.defineProcessMetaData(
+         TestUtils.TABLE_NAME_SHAPE,
+         TestUtils.TABLE_NAME_PERSON_MEMORY,
+         ExtractViaQueryStep.class,
+         StreamedETLWithFrontendProcessTest.TestTransformShapeToPersonStep.class,
+         LoadViaInsertStep.class).withBasepullConfiguration(new BasepullConfiguration()), qInstance, validator);
+      assertValidationFailureReasons(false, validator.getErrors(), "does not have a field with a default value that is a BasepullExtractStepInterface CodeReference");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testExtractViaBasepullQueryPasses()
+   {
+      QInstance          qInstance = QContext.getQInstance();
+      QInstanceValidator validator = new QInstanceValidator();
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      // set up a streamed-etl process, with an ExtractViaBasepullQueryStep as expected - should pass //
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      new BasepullExtractStepValidator().validate(StreamedETLWithFrontendProcess.defineProcessMetaData(
+         TestUtils.TABLE_NAME_SHAPE,
+         TestUtils.TABLE_NAME_PERSON_MEMORY,
+         ExtractViaBasepullQueryStep.class,
+         StreamedETLWithFrontendProcessTest.TestTransformShapeToPersonStep.class,
+         LoadViaInsertStep.class).withBasepullConfiguration(new BasepullConfiguration()), qInstance, validator);
+      assertThat(validator.getErrors()).isNullOrEmpty();
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QCollectingLoggerTest.java

@@ -0,0 +1,92 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.logging;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import org.apache.logging.log4j.Level;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for QCollectingLogger 
+ *******************************************************************************/
+class QCollectingLoggerTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test()
+   {
+      ClassThatLogsThings classThatLogsThings = new ClassThatLogsThings();
+      classThatLogsThings.logAnInfo("1");
+
+      QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(ClassThatLogsThings.class);
+      classThatLogsThings.logAnInfo("2");
+      classThatLogsThings.logAWarn("3");
+      QLogger.deactivateCollectingLoggerForClass(ClassThatLogsThings.class);
+
+      classThatLogsThings.logAWarn("4");
+
+      assertEquals(2, collectingLogger.getCollectedMessages().size());
+
+      assertThat(collectingLogger.getCollectedMessages().get(0).getMessage()).contains("""
+         "message":"2",""");
+      assertEquals("2", collectingLogger.getCollectedMessages().get(0).getMessageAsJSONObject().getString("message"));
+      assertEquals(Level.INFO, collectingLogger.getCollectedMessages().get(0).getLevel());
+
+      assertThat(collectingLogger.getCollectedMessages().get(1).getMessage()).contains("""
+         "message":"3",""");
+      assertEquals(Level.WARN, collectingLogger.getCollectedMessages().get(1).getLevel());
+      assertEquals("3", collectingLogger.getCollectedMessages().get(1).getMessageAsJSONObject().getString("message"));
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class ClassThatLogsThings
+   {
+      private static final QLogger LOG = QLogger.getLogger(ClassThatLogsThings.class);
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      private void logAnInfo(String message)
+      {
+         LOG.info(message);
+      }
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      private void logAWarn(String message)
+      {
+         LOG.warn(message);
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java

@@ -29,13 +29,20 @@ import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
+import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
@@ -74,6 +81,43 @@ class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    ** at one point, when the review step go added, we were double-adding records
+    ** to the output/result screen.  This test verifies, if we run the full process
+    ** that that doesn't happen.
+    *******************************************************************************/
+   @Test
+   void testTwoFailedUpdatesFullProcess() throws QException
+   {
+      QContext.getQInstance().addProcess(new HealBadRecordAutomationStatusesProcessStep().produce(QContext.getQInstance()));
+
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(new QRecord(), new QRecord())));
+      List<QRecord> records = queryAllRecords();
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records, AutomationStatus.FAILED_UPDATE_AUTOMATIONS, null);
+
+      assertThat(queryAllRecords()).allMatch(r -> AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName(HealBadRecordAutomationStatusesProcessStep.NAME);
+      input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, records.stream().map(r -> r.getValue("id")).toList()))));
+      RunProcessAction runProcessAction = new RunProcessAction();
+      RunProcessOutput runProcessOutput = runProcessAction.execute(input);
+
+      input.setStartAfterStep(runProcessOutput.getProcessState().getNextStepName().get());
+      runProcessOutput = runProcessAction.execute(input);
+
+      input.setStartAfterStep(runProcessOutput.getProcessState().getNextStepName().get());
+      runProcessOutput = runProcessAction.execute(input);
+
+      List<QRecord> outputRecords = runProcessOutput.getProcessState().getRecords();
+      assertEquals(1, outputRecords.size());
+      assertEquals(2, outputRecords.get(0).getValueInteger("count"));
+
+      assertThat(queryAllRecords()).allMatch(r -> AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/