From 01d3937889efe78e7b0497329480239dfe55500d Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 16 May 2023 11:38:24 -0500 Subject: [PATCH 1/2] hotfix: attempt to remove some caching bugs when updating old cache records and we got uncachable errors --- .../core/actions/tables/GetAction.java | 89 ++++++++++++------- 1 file changed, 58 insertions(+), 31 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java index 0d16b5c9..f86e8f3f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java @@ -129,29 +129,14 @@ public class GetAction /////////////////////////////////////////////////////////////////////// // if the record wasn't found, see if we should look in cache-source // /////////////////////////////////////////////////////////////////////// - QRecord recordFromSource = tryToGetFromCacheSource(getInput, getOutput); + QRecord recordFromSource = tryToGetFromCacheSource(getInput); if(recordFromSource != null) { + ///////////////////////////////////////////////////////////////////////////////////////////////// + // good, we found a record from the source, make sure we should cache it, and if so, do it now // + ///////////////////////////////////////////////////////////////////////////////////////////////// QRecord recordToCache = mapSourceRecordToCacheRecord(table, recordFromSource); - boolean shouldCacheRecord = true; - - //////////////////////////////////////////////////////////////////////////////// - // see if there are any exclustions that need to be considered for this table // - //////////////////////////////////////////////////////////////////////////////// - recordMatchExclusionLoop: - for(CacheUseCase useCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases())) - { - for(QQueryFilter filter : CollectionUtils.nonNullList(useCase.getExcludeRecordsMatching())) - { - if(BackendQueryFilterUtils.doesRecordMatch(filter, recordToCache)) - { - LOG.info("Not caching record because it matches a use case's filter exclusion", new LogPair("record", recordToCache), new LogPair("filter", filter)); - shouldCacheRecord = false; - break recordMatchExclusionLoop; - } - } - } - + boolean shouldCacheRecord = shouldCacheRecord(table, recordToCache); if(shouldCacheRecord) { InsertInput insertInput = new InsertInput(); @@ -184,12 +169,50 @@ public class GetAction + /******************************************************************************* + ** + *******************************************************************************/ + private boolean shouldCacheRecord(QTableMetaData table, QRecord recordToCache) + { + boolean shouldCacheRecord = true; + recordMatchExclusionLoop: + for(CacheUseCase useCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases())) + { + for(QQueryFilter filter : CollectionUtils.nonNullList(useCase.getExcludeRecordsMatching())) + { + if(BackendQueryFilterUtils.doesRecordMatch(filter, recordToCache)) + { + LOG.info("Not caching record because it matches a use case's filter exclusion", new LogPair("record", recordToCache), new LogPair("filter", filter)); + shouldCacheRecord = false; + break recordMatchExclusionLoop; + } + } + } + + return (shouldCacheRecord); + } + + + /******************************************************************************* ** *******************************************************************************/ private static QRecord mapSourceRecordToCacheRecord(QTableMetaData table, QRecord recordFromSource) { QRecord cacheRecord = new QRecord(recordFromSource); + + ////////////////////////////////////////////////////////////////////////////////////////////// + // make sure every value in the qRecord is set, because we will possibly be doing an update // + // on this record and want to null out any fields not set, not leave them populated // + ////////////////////////////////////////////////////////////////////////////////////////////// + for(String fieldName : table.getFields().keySet()) + { + if(!cacheRecord.getValues().containsKey(fieldName)) + { + cacheRecord.setValue(fieldName, null); + } + } + if(StringUtils.hasContent(table.getCacheOf().getCachedDateFieldName())) { cacheRecord.setValue(table.getCacheOf().getCachedDateFieldName(), Instant.now()); @@ -212,7 +235,8 @@ public class GetAction Instant cachedDate = cachedRecord.getValueInstant(table.getCacheOf().getCachedDateFieldName()); if(cachedDate == null || cachedDate.isBefore(Instant.now().minus(expirationSeconds, ChronoUnit.SECONDS))) { - QRecord recordFromSource = tryToGetFromCacheSource(getInput, getOutput); + QRecord recordFromSource = tryToGetFromCacheSource(getInput); + boolean shouldDeleteCachedRecord = true; if(recordFromSource != null) { /////////////////////////////////////////////////////////////////// @@ -220,15 +244,20 @@ public class GetAction /////////////////////////////////////////////////////////////////// QRecord recordToCache = mapSourceRecordToCacheRecord(table, recordFromSource); recordToCache.setValue(table.getPrimaryKeyField(), cachedRecord.getValue(table.getPrimaryKeyField())); + getOutput.setRecord(recordToCache); - UpdateInput updateInput = new UpdateInput(); - updateInput.setTableName(getInput.getTableName()); - updateInput.setRecords(List.of(recordToCache)); - UpdateOutput updateOutput = new UpdateAction().execute(updateInput); - - getOutput.setRecord(updateOutput.getRecords().get(0)); + if(shouldCacheRecord(table, recordToCache)) + { + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(getInput.getTableName()); + updateInput.setRecords(List.of(recordToCache)); + UpdateOutput updateOutput = new UpdateAction().execute(updateInput); + getOutput.setRecord(updateOutput.getRecords().get(0)); + shouldDeleteCachedRecord = false; + } } - else + + if(shouldDeleteCachedRecord) { ///////////////////////////////////////////////////////////////////////////// // if the record is no longer in the source, then remove it from the cache // @@ -237,8 +266,6 @@ public class GetAction deleteInput.setTableName(getInput.getTableName()); deleteInput.setPrimaryKeys(List.of(getOutput.getRecord().getValue(table.getPrimaryKeyField()))); new DeleteAction().execute(deleteInput); - - getOutput.setRecord(null); } } } @@ -249,7 +276,7 @@ public class GetAction /******************************************************************************* ** *******************************************************************************/ - private QRecord tryToGetFromCacheSource(GetInput getInput, GetOutput getOutput) throws QException + private QRecord tryToGetFromCacheSource(GetInput getInput) throws QException { QRecord recordFromSource = null; QTableMetaData table = getInput.getTable(); From 3f9370e9b5755f789061b078a8fd06be9999a7aa Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 16 May 2023 12:02:47 -0500 Subject: [PATCH 2/2] hotfix: updated to null out the getoutput's record if it was not returned by the original source --- .../core/actions/tables/GetAction.java | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java index f86e8f3f..8a2586ec 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java @@ -235,13 +235,23 @@ public class GetAction Instant cachedDate = cachedRecord.getValueInstant(table.getCacheOf().getCachedDateFieldName()); if(cachedDate == null || cachedDate.isBefore(Instant.now().minus(expirationSeconds, ChronoUnit.SECONDS))) { - QRecord recordFromSource = tryToGetFromCacheSource(getInput); - boolean shouldDeleteCachedRecord = true; + ////////////////////////////////////////////////////////////////////////// + // keep the serial key from the old record in case we need to delete it // + ////////////////////////////////////////////////////////////////////////// + Serializable oldRecordPrimaryKey = getOutput.getRecord().getValue(table.getPrimaryKeyField()); + boolean shouldDeleteCachedRecord = true; + + /////////////////////////////////////////// + // fetch record from original source now // + /////////////////////////////////////////// + QRecord recordFromSource = tryToGetFromCacheSource(getInput); if(recordFromSource != null) { - /////////////////////////////////////////////////////////////////// - // if the record was found in the source, update it in the cache // - /////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////// + // if the record was found in the source, put it into the output // + // object so returned back to caller, check that it should actually // + // be cached before doing so // + ////////////////////////////////////////////////////////////////////// QRecord recordToCache = mapSourceRecordToCacheRecord(table, recordFromSource); recordToCache.setValue(table.getPrimaryKeyField(), cachedRecord.getValue(table.getPrimaryKeyField())); getOutput.setRecord(recordToCache); @@ -256,6 +266,13 @@ public class GetAction shouldDeleteCachedRecord = false; } } + else + { + /////////////////////////////////////////////////////////////////////////////////////// + // if we did not get a record back from the source, empty out the getOutput's record // + /////////////////////////////////////////////////////////////////////////////////////// + getOutput.setRecord(null); + } if(shouldDeleteCachedRecord) { @@ -264,7 +281,7 @@ public class GetAction ///////////////////////////////////////////////////////////////////////////// DeleteInput deleteInput = new DeleteInput(); deleteInput.setTableName(getInput.getTableName()); - deleteInput.setPrimaryKeys(List.of(getOutput.getRecord().getValue(table.getPrimaryKeyField()))); + deleteInput.setPrimaryKeys(List.of(oldRecordPrimaryKey)); new DeleteAction().execute(deleteInput); } }