Merged feature/CE-680-push-tracking-data-to-ship-station into feature/join-enhancements

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java

@@ -28,6 +28,9 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
@@ -52,6 +55,41 @@ public class AuditSingleInput
 
 
 
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public AuditSingleInput()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public AuditSingleInput(QTableMetaData table, QRecord record, String auditMessage)
+   {
+      setAuditTableName(table.getName());
+      setRecordId(record.getValueInteger(table.getPrimaryKeyField()));
+      setSecurityKeyValues(AuditAction.getRecordSecurityKeyValues(table, record, Optional.empty()));
+      setMessage(auditMessage);
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public AuditSingleInput(String tableName, QRecord record, String auditMessage)
+   {
+      this(QContext.getQInstance().getTable(tableName), record, auditMessage);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for auditTableName
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java

@@ -89,6 +89,34 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
 
 
 
+   /*******************************************************************************
+    ** Let a subclass know if getQueryFilter will use the "default filter" (e.g., from
+    ** our base class, which would come from values passed in to the process), or if
+    ** the BasePull Query would be used (e.g., for a scheduled job).
+    *******************************************************************************/
+   protected boolean willTheBasePullQueryBeUsed(RunBackendStepInput runBackendStepInput)
+   {
+      try
+      {
+         super.getQueryFilter(runBackendStepInput);
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if super.getQueryFilter returned - then - there's a default query to use (e.g., a user selecting rows on a screen). //
+         // this means we won't use the BasePull query, so return a false here.                                                 //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         return (false);
+      }
+      catch(QException qe)
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if we catch here, assume that is because there was no default filter - in which case - we'll use the BasePull Query //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         return (true);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java

@@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -92,8 +93,9 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
       .withSingularPastMessage("was not synced, because it is ")
       .withPluralPastMessage("were not synced, because they are ");
 
-   protected RunBackendStepInput runBackendStepInput = null;
+   protected RunBackendStepInput  runBackendStepInput  = null;
-   protected RecordLookupHelper  recordLookupHelper  = null;
+   protected RunBackendStepOutput runBackendStepOutput = null;
+   protected RecordLookupHelper   recordLookupHelper   = null;
 
    private QPossibleValueTranslator possibleValueTranslator;
 
@@ -105,15 +107,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
    @Override
    public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
    {
-      ArrayList<ProcessSummaryLineInterface> processSummaryLineList = StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate, errorMissingKeyField);
+      ArrayList<ProcessSummaryLineInterface> processSummaryLineList = StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate, errorMissingKeyField, willNotInsert, willNotUpdate);
-      if(willNotInsert.getCount() > 0)
-      {
-         processSummaryLineList.add(willNotInsert);
-      }
-      if(willNotUpdate.getCount() > 0)
-      {
-         processSummaryLineList.add(willNotUpdate);
-      }
       return (processSummaryLineList);
    }
 
@@ -193,6 +187,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
       }
 
       this.runBackendStepInput = runBackendStepInput;
+      this.runBackendStepOutput = runBackendStepOutput;
 
       SyncProcessConfig config = getSyncProcessConfig();
 
@@ -277,12 +272,10 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
          if(existingRecord != null && config.performUpdates)
          {
             recordToStore = existingRecord;
-            okToUpdate.incrementCount();
          }
          else if(existingRecord == null && config.performInserts)
          {
             recordToStore = new QRecord();
-            okToInsert.incrementCount();
          }
          else
          {
@@ -299,12 +292,21 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
             continue;
          }
 
-         ////////////////////////////////////////////////////////////////
+         //////////////////////////////////////////////////////////////////////////////////
-         // if we received a record to store add to the output records //
+         // if we received a record to store add to the output records and summary lines //
-         ////////////////////////////////////////////////////////////////
+         //////////////////////////////////////////////////////////////////////////////////
          recordToStore = populateRecordToStore(runBackendStepInput, recordToStore, sourceRecord);
          if(recordToStore != null)
          {
+            if(existingRecord != null)
+            {
+               okToUpdate.incrementCount();
+            }
+            else
+            {
+               okToInsert.incrementCount();
+            }
+
             runBackendStepOutput.addRecord(recordToStore);
          }
       }
@@ -426,4 +428,17 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
       }
    }
 
+
+
+   /*******************************************************************************
+    ** Let the subclass "easily" add an audit to be inserted on the Execute step.
+    *******************************************************************************/
+   protected void addAuditForExecuteStep(AuditSingleInput auditSingleInput)
+   {
+      if(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE.equals(this.runBackendStepInput.getStepName()))
+      {
+         this.runBackendStepOutput.addAuditSingleInput(auditSingleInput);
+      }
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStepTest.java

@@ -23,14 +23,17 @@ package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
 
 
 import java.time.Instant;
+import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
@@ -71,4 +74,27 @@ class ExtractViaBasepullQueryStepTest extends BaseTest
       assertTrue(queryFilter.getOrderBys().get(0).getIsAscending());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testWillTheBasePullQueryBeUsed()
+   {
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // only time the base-pull query will be used is if there isn't a filter or records in the process input. //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertTrue(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput()));
+
+      assertFalse(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput()
+         .withValues(Map.of("recordIds", "1,2,3", StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, "person"))));
+
+      assertFalse(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput()
+         .withValues(Map.of(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter()))));
+
+      assertFalse(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput()
+         .withValues(Map.of("queryFilterJson", "{}"))));
+   }
+
 }