QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 21:50:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merged feature/CE-680-push-tracking-data-to-ship-station into feature/join-enhancements

Browse Source
This commit is contained in:
darin.kelkhoff
2023-09-27 20:00:42 -05:00
parent bf0a554c6a 7160b87048
commit 923872dab7
4 changed files with 123 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java
View File

@ -28,6 +28,9 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
@ -52,6 +55,41 @@ public class AuditSingleInput
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public AuditSingleInput()
{
}
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public AuditSingleInput(QTableMetaData table, QRecord record, String auditMessage)
{
setAuditTableName(table.getName());
setRecordId(record.getValueInteger(table.getPrimaryKeyField()));
setSecurityKeyValues(AuditAction.getRecordSecurityKeyValues(table, record, Optional.empty()));
setMessage(auditMessage);
}
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public AuditSingleInput(String tableName, QRecord record, String auditMessage)
{
this(QContext.getQInstance().getTable(tableName), record, auditMessage);
}
/*******************************************************************************
** Getter for auditTableName
*******************************************************************************/

28
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java
View File

@ -89,6 +89,34 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
/*******************************************************************************
** Let a subclass know if getQueryFilter will use the "default filter" (e.g., from
** our base class, which would come from values passed in to the process), or if
** the BasePull Query would be used (e.g., for a scheduled job).
*******************************************************************************/
protected boolean willTheBasePullQueryBeUsed(RunBackendStepInput runBackendStepInput)
{
try
{
super.getQueryFilter(runBackendStepInput);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if super.getQueryFilter returned - then - there's a default query to use (e.g., a user selecting rows on a screen). //
// this means we won't use the BasePull query, so return a false here. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
return (false);
}
catch(QException qe)
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if we catch here, assume that is because there was no default filter - in which case - we'll use the BasePull Query //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
return (true);
}
}
/*******************************************************************************
**
*******************************************************************************/

47
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java
View File

@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@ -92,8 +93,9 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
.withSingularPastMessage("was not synced, because it is ")
.withPluralPastMessage("were not synced, because they are ");
protected RunBackendStepInput runBackendStepInput = null;
protected RecordLookupHelper recordLookupHelper = null;
protected RunBackendStepInput runBackendStepInput = null;
protected RunBackendStepOutput runBackendStepOutput = null;
protected RecordLookupHelper recordLookupHelper = null;
private QPossibleValueTranslator possibleValueTranslator;
@ -105,15 +107,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
{
ArrayList<ProcessSummaryLineInterface> processSummaryLineList = StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate, errorMissingKeyField);
if(willNotInsert.getCount() > 0)
{
processSummaryLineList.add(willNotInsert);
}
if(willNotUpdate.getCount() > 0)
{
processSummaryLineList.add(willNotUpdate);
}
ArrayList<ProcessSummaryLineInterface> processSummaryLineList = StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate, errorMissingKeyField, willNotInsert, willNotUpdate);
return (processSummaryLineList);
}
@ -193,6 +187,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
}
this.runBackendStepInput = runBackendStepInput;
this.runBackendStepOutput = runBackendStepOutput;
SyncProcessConfig config = getSyncProcessConfig();
@ -277,12 +272,10 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
if(existingRecord != null && config.performUpdates)
{
recordToStore = existingRecord;
okToUpdate.incrementCount();
}
else if(existingRecord == null && config.performInserts)
{
recordToStore = new QRecord();
okToInsert.incrementCount();
}
else
{
@ -299,12 +292,21 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
continue;
}
////////////////////////////////////////////////////////////////
// if we received a record to store add to the output records //
////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////
// if we received a record to store add to the output records and summary lines //
//////////////////////////////////////////////////////////////////////////////////
recordToStore = populateRecordToStore(runBackendStepInput, recordToStore, sourceRecord);
if(recordToStore != null)
{
if(existingRecord != null)
{
okToUpdate.incrementCount();
}
else
{
okToInsert.incrementCount();
}
runBackendStepOutput.addRecord(recordToStore);
}
}
@ -426,4 +428,17 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
}
}
/*******************************************************************************
** Let the subclass "easily" add an audit to be inserted on the Execute step.
*******************************************************************************/
protected void addAuditForExecuteStep(AuditSingleInput auditSingleInput)
{
if(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE.equals(this.runBackendStepInput.getStepName()))
{
this.runBackendStepOutput.addAuditSingleInput(auditSingleInput);
}
}
}

26
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStepTest.java
View File

@ -23,14 +23,17 @@ package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
import java.time.Instant;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -71,4 +74,27 @@ class ExtractViaBasepullQueryStepTest extends BaseTest
assertTrue(queryFilter.getOrderBys().get(0).getIsAscending());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWillTheBasePullQueryBeUsed()
{
////////////////////////////////////////////////////////////////////////////////////////////////////////////
// only time the base-pull query will be used is if there isn't a filter or records in the process input. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////
assertTrue(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput()));
assertFalse(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput()
.withValues(Map.of("recordIds", "1,2,3", StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, "person"))));
assertFalse(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput()
.withValues(Map.of(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter()))));
assertFalse(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput()
.withValues(Map.of("queryFilterJson", "{}"))));
}
}