diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java index 655d31d0..0cbf6a65 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java @@ -28,6 +28,9 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.audits.AuditAction; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters; @@ -52,6 +55,41 @@ public class AuditSingleInput + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public AuditSingleInput() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public AuditSingleInput(QTableMetaData table, QRecord record, String auditMessage) + { + setAuditTableName(table.getName()); + setRecordId(record.getValueInteger(table.getPrimaryKeyField())); + setSecurityKeyValues(AuditAction.getRecordSecurityKeyValues(table, record, Optional.empty())); + setMessage(auditMessage); + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public AuditSingleInput(String tableName, QRecord record, String auditMessage) + { + this(QContext.getQInstance().getTable(tableName), record, auditMessage); + } + + + /******************************************************************************* ** Getter for auditTableName *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java index 722885ec..266df3fe 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java @@ -89,6 +89,34 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep + /******************************************************************************* + ** Let a subclass know if getQueryFilter will use the "default filter" (e.g., from + ** our base class, which would come from values passed in to the process), or if + ** the BasePull Query would be used (e.g., for a scheduled job). + *******************************************************************************/ + protected boolean willTheBasePullQueryBeUsed(RunBackendStepInput runBackendStepInput) + { + try + { + super.getQueryFilter(runBackendStepInput); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if super.getQueryFilter returned - then - there's a default query to use (e.g., a user selecting rows on a screen). // + // this means we won't use the BasePull query, so return a false here. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + return (false); + } + catch(QException qe) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we catch here, assume that is because there was no default filter - in which case - we'll use the BasePull Query // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + return (true); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index 71769499..27c47ba5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -92,8 +93,9 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt .withSingularPastMessage("was not synced, because it is ") .withPluralPastMessage("were not synced, because they are "); - protected RunBackendStepInput runBackendStepInput = null; - protected RecordLookupHelper recordLookupHelper = null; + protected RunBackendStepInput runBackendStepInput = null; + protected RunBackendStepOutput runBackendStepOutput = null; + protected RecordLookupHelper recordLookupHelper = null; private QPossibleValueTranslator possibleValueTranslator; @@ -105,15 +107,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt @Override public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) { - ArrayList processSummaryLineList = StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate, errorMissingKeyField); - if(willNotInsert.getCount() > 0) - { - processSummaryLineList.add(willNotInsert); - } - if(willNotUpdate.getCount() > 0) - { - processSummaryLineList.add(willNotUpdate); - } + ArrayList processSummaryLineList = StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate, errorMissingKeyField, willNotInsert, willNotUpdate); return (processSummaryLineList); } @@ -193,6 +187,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt } this.runBackendStepInput = runBackendStepInput; + this.runBackendStepOutput = runBackendStepOutput; SyncProcessConfig config = getSyncProcessConfig(); @@ -277,12 +272,10 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt if(existingRecord != null && config.performUpdates) { recordToStore = existingRecord; - okToUpdate.incrementCount(); } else if(existingRecord == null && config.performInserts) { recordToStore = new QRecord(); - okToInsert.incrementCount(); } else { @@ -299,12 +292,21 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt continue; } - //////////////////////////////////////////////////////////////// - // if we received a record to store add to the output records // - //////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////////// + // if we received a record to store add to the output records and summary lines // + ////////////////////////////////////////////////////////////////////////////////// recordToStore = populateRecordToStore(runBackendStepInput, recordToStore, sourceRecord); if(recordToStore != null) { + if(existingRecord != null) + { + okToUpdate.incrementCount(); + } + else + { + okToInsert.incrementCount(); + } + runBackendStepOutput.addRecord(recordToStore); } } @@ -426,4 +428,17 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt } } + + + /******************************************************************************* + ** Let the subclass "easily" add an audit to be inserted on the Execute step. + *******************************************************************************/ + protected void addAuditForExecuteStep(AuditSingleInput auditSingleInput) + { + if(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE.equals(this.runBackendStepInput.getStepName())) + { + this.runBackendStepOutput.addAuditSingleInput(auditSingleInput); + } + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStepTest.java index fe2bca51..690b290e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStepTest.java @@ -23,14 +23,17 @@ package com.kingsrook.qqq.backend.core.processes.implementations.basepull; import java.time.Instant; +import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -71,4 +74,27 @@ class ExtractViaBasepullQueryStepTest extends BaseTest assertTrue(queryFilter.getOrderBys().get(0).getIsAscending()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWillTheBasePullQueryBeUsed() + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // only time the base-pull query will be used is if there isn't a filter or records in the process input. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertTrue(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput())); + + assertFalse(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput() + .withValues(Map.of("recordIds", "1,2,3", StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, "person")))); + + assertFalse(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput() + .withValues(Map.of(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter())))); + + assertFalse(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput() + .withValues(Map.of("queryFilterJson", "{}")))); + } + }