From b687d07e46a7758c44e2b9fe5e6b5b68faf83757 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Fri, 4 Oct 2024 12:24:58 -0500 Subject: [PATCH 1/5] CE-1836: update abstract table sync to make members and functions protected --- .../AbstractTableSyncTransformStep.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index 68619a28..3eaba3e2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -72,33 +72,33 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; *******************************************************************************/ public abstract class AbstractTableSyncTransformStep extends AbstractTransformStep { - private static final QLogger LOG = QLogger.getLogger(AbstractTableSyncTransformStep.class); + protected static final QLogger LOG = QLogger.getLogger(AbstractTableSyncTransformStep.class); - private ProcessSummaryLine okToInsert = StandardProcessSummaryLineProducer.getOkToInsertLine(); - private ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine(); + protected ProcessSummaryLine okToInsert = StandardProcessSummaryLineProducer.getOkToInsertLine(); + protected ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine(); - private ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO) + protected ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO) .withMessageSuffix("because this process is not configured to insert records.") .withSingularFutureMessage("will not be inserted ") .withPluralFutureMessage("will not be inserted ") .withSingularPastMessage("was not inserted ") .withPluralPastMessage("were not inserted "); - private ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO) + protected ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO) .withMessageSuffix("because this process is not configured to update records.") .withSingularFutureMessage("will not be updated ") .withPluralFutureMessage("will not be updated ") .withSingularPastMessage("was not updated ") .withPluralPastMessage("were not updated "); - private ProcessSummaryLine errorMissingKeyField = new ProcessSummaryLine(Status.ERROR) + protected ProcessSummaryLine errorMissingKeyField = new ProcessSummaryLine(Status.ERROR) .withMessageSuffix("missing a value for the key field.") .withSingularFutureMessage("will not be synced, because it is ") .withPluralFutureMessage("will not be synced, because they are ") .withSingularPastMessage("was not synced, because it is ") .withPluralPastMessage("were not synced, because they are "); - private ProcessSummaryLine unspecifiedError = new ProcessSummaryLine(Status.ERROR) + protected ProcessSummaryLine unspecifiedError = new ProcessSummaryLine(Status.ERROR) .withMessageSuffix("of an unexpected error: ") .withSingularFutureMessage("will not be synced, ") .withPluralFutureMessage("will not be synced, ") @@ -109,7 +109,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt protected RunBackendStepOutput runBackendStepOutput = null; protected RecordLookupHelper recordLookupHelper = null; - private QPossibleValueTranslator possibleValueTranslator; + protected QPossibleValueTranslator possibleValueTranslator; @@ -374,6 +374,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt } + /******************************************************************************* ** Given a source record, extract what we'll use as its key from it. ** From 4f92fb2ae230d8879d11ec0eb24e3e01092e4201 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Mon, 7 Oct 2024 22:33:16 -0500 Subject: [PATCH 2/5] CE-1836: updates to allow getting basepull key value and sync config perform insert/updates from input --- .../actions/processes/RunProcessAction.java | 11 ++++-- .../AbstractTableSyncTransformStep.java | 35 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index 03ee26aa..ada00f9a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -79,6 +79,7 @@ public class RunProcessAction { private static final QLogger LOG = QLogger.getLogger(RunProcessAction.class); + public static final String BASEPULL_KEY_VALUE = "basepullKeyValue"; public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey"; public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey"; public static final String BASEPULL_TIMESTAMP_FIELD = "basepullTimestampField"; @@ -517,9 +518,13 @@ public class RunProcessAction /******************************************************************************* ** *******************************************************************************/ - protected String determineBasepullKeyValue(QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException + protected String determineBasepullKeyValue(QProcessMetaData process, RunProcessInput runProcessInput, BasepullConfiguration basepullConfiguration) throws QException { String basepullKeyValue = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName(); + if(runProcessInput.getValueString(BASEPULL_KEY_VALUE) != null) + { + basepullKeyValue = runProcessInput.getValueString(BASEPULL_KEY_VALUE); + } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if process specifies that it uses variants, look for that data in the session and append to our basepull key // @@ -551,7 +556,7 @@ public class RunProcessAction String basepullTableName = basepullConfiguration.getTableName(); String basepullKeyFieldName = basepullConfiguration.getKeyField(); String basepullLastRunTimeFieldName = basepullConfiguration.getLastRunTimeFieldName(); - String basepullKeyValue = determineBasepullKeyValue(process, basepullConfiguration); + String basepullKeyValue = determineBasepullKeyValue(process, runProcessInput, basepullConfiguration); /////////////////////////////////////// // get the stored basepull timestamp // @@ -631,7 +636,7 @@ public class RunProcessAction String basepullKeyFieldName = basepullConfiguration.getKeyField(); String basepullLastRunTimeFieldName = basepullConfiguration.getLastRunTimeFieldName(); Integer basepullHoursBackForInitialTimestamp = basepullConfiguration.getHoursBackForInitialTimestamp(); - String basepullKeyValue = determineBasepullKeyValue(process, basepullConfiguration); + String basepullKeyValue = determineBasepullKeyValue(process, runProcessInput, basepullConfiguration); /////////////////////////////////////// // get the stored basepull timestamp // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index 3eaba3e2..fed50bad 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -111,6 +111,9 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt protected QPossibleValueTranslator possibleValueTranslator; + protected static final String SYNC_TABLE_PERFORM_INSERTS_KEY = "syncTablePerformInsertsKey"; + protected static final String SYNC_TABLE_PERFORM_UPDATES_KEY = "syncTablePerformUpdatesKey"; + /******************************************************************************* @@ -193,6 +196,26 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt + /******************************************************************************* + ** + *******************************************************************************/ + void setPerformInserts(boolean performInserts) + { + this.setPerformInserts(performInserts); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + void setPerformUpdates(boolean performUpdates) + { + this.setPerformUpdates(performUpdates); + } + + + /******************************************************************************* ** artificial method, here to make jacoco see that this class is indeed ** included in test coverage... @@ -222,6 +245,18 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt SyncProcessConfig config = getSyncProcessConfig(); + //////////////////////////////////////////////////////////// + // see if these fields have been updated via input fields // + //////////////////////////////////////////////////////////// + if(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY) != null) + { + config.setPerformInserts(Boolean.parseBoolean(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY))); + } + if(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_UPDATES_KEY) != null) + { + config.setPerformUpdates(Boolean.parseBoolean(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_UPDATES_KEY))); + } + String sourceTableKeyField = config.sourceTableKeyField; String destinationTableForeignKeyField = config.destinationTableForeignKey; String destinationTableName = config.destinationTable; From 526ba6ca307c727891758eb96aae9cd53acc6467 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 8 Oct 2024 15:46:47 -0500 Subject: [PATCH 3/5] CE-1836: added potential to log output --- .../AbstractTableSyncTransformStep.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index fed50bad..920d5090 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.tablesync; import java.io.Serializable; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -35,6 +37,7 @@ import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -53,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.processes.implementations.general.StandardProcessSummaryLineProducer; @@ -113,6 +117,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt protected static final String SYNC_TABLE_PERFORM_INSERTS_KEY = "syncTablePerformInsertsKey"; protected static final String SYNC_TABLE_PERFORM_UPDATES_KEY = "syncTablePerformUpdatesKey"; + protected static final String LOG_TRANSFORM_RESULTS = "logTransformResults"; @@ -237,6 +242,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt { if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords())) { + LOG.info("No input records were found."); return; } @@ -406,6 +412,53 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt possibleValueTranslator.translatePossibleValuesInRecords(QContext.getQInstance().getTable(destinationTableName), runBackendStepOutput.getRecords()); } } + + if(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY) != null) + { + logResults(runBackendStepInput, config); + } + } + + + + /******************************************************************************* + ** Log results of transformation + ** + *******************************************************************************/ + protected void logResults(RunBackendStepInput runBackendStepInput, SyncProcessConfig syncProcessConfig) + { + String timezone = QContext.getQSession().getValue(QSession.VALUE_KEY_USER_TIMEZONE); + if(timezone == null) + { + timezone = QContext.getQInstance().getDefaultTimeZoneId(); + } + ZonedDateTime dateTime = runBackendStepInput.getBasepullLastRunTime().atZone(ZoneId.of(timezone)); + + if(syncProcessConfig.performInserts) + { + if(okToInsert.getCount() == 0) + { + LOG.info("No Records were found to insert since " + QValueFormatter.formatDateTimeWithZone(dateTime) + "."); + } + else + { + String pluralized = okToInsert.getCount() > 1 ? " Records were " : " Record was "; + LOG.info(okToInsert.getCount() + pluralized + " found to insert since " + QValueFormatter.formatDateTimeWithZone(dateTime) + ".", logPair("primaryKeys", okToInsert.getPrimaryKeys())); + } + } + + if(syncProcessConfig.performUpdates) + { + if(okToUpdate.getCount() == 0) + { + LOG.info("No Records were found to update since " + QValueFormatter.formatDateTimeWithZone(dateTime) + "."); + } + else + { + String pluralized = okToUpdate.getCount() > 1 ? " Records were " : " Record was "; + LOG.info(okToUpdate.getCount() + pluralized + " found to update since " + QValueFormatter.formatDateTimeWithZone(dateTime) + ".", logPair("primaryKeys", okToInsert.getPrimaryKeys())); + } + } } From 10014f16ae42e945703dd8a35403aa02d8d39630 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 8 Oct 2024 16:20:23 -0500 Subject: [PATCH 4/5] CE-1836: fixed to check as boolean --- .../tablesync/AbstractTableSyncTransformStep.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index 920d5090..7168321a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -262,7 +262,6 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt { config.setPerformUpdates(Boolean.parseBoolean(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_UPDATES_KEY))); } - String sourceTableKeyField = config.sourceTableKeyField; String destinationTableForeignKeyField = config.destinationTableForeignKey; String destinationTableName = config.destinationTable; @@ -413,7 +412,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt } } - if(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY) != null) + if(Boolean.parseBoolean(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY))) { logResults(runBackendStepInput, config); } From e0597827ef8b4d4c83c488651a155df0df673624 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Wed, 9 Oct 2024 10:30:49 -0500 Subject: [PATCH 5/5] CE-1836: updates from code review --- .../AbstractTableSyncTransformStep.java | 26 +++---------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index 7168321a..9b0934a0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -201,26 +201,6 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt - /******************************************************************************* - ** - *******************************************************************************/ - void setPerformInserts(boolean performInserts) - { - this.setPerformInserts(performInserts); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - void setPerformUpdates(boolean performUpdates) - { - this.setPerformUpdates(performUpdates); - } - - - /******************************************************************************* ** artificial method, here to make jacoco see that this class is indeed ** included in test coverage... @@ -256,11 +236,11 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt //////////////////////////////////////////////////////////// if(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY) != null) { - config.setPerformInserts(Boolean.parseBoolean(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY))); + config = new SyncProcessConfig(config.sourceTable, config.sourceTableKeyField, config.destinationTable, config.destinationTableForeignKey, true, config.performUpdates); } if(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_UPDATES_KEY) != null) { - config.setPerformUpdates(Boolean.parseBoolean(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_UPDATES_KEY))); + config = new SyncProcessConfig(config.sourceTable, config.sourceTableKeyField, config.destinationTable, config.destinationTableForeignKey, config.performUpdates, true); } String sourceTableKeyField = config.sourceTableKeyField; String destinationTableForeignKeyField = config.destinationTableForeignKey; @@ -412,7 +392,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt } } - if(Boolean.parseBoolean(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY))) + if(Boolean.parseBoolean(runBackendStepInput.getValueString(LOG_TRANSFORM_RESULTS))) { logResults(runBackendStepInput, config); }