QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor caching out of GetAction - namely, to support initial use-cases in QueryAction.

Browse Source
This commit is contained in:
darin.kelkhoff
2023-06-30 12:36:15 -05:00
parent 75ae848afd
commit c086874e64
9 changed files with 2099 additions and 431 deletions
Download Patch File Download Diff File Expand all files Collapse all files

301
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java
View File

@ -23,8 +23,6 @@ package com.kingsrook.qqq.backend.core.actions.tables;
import java.io.Serializable; import java.io.Serializable;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
@ -33,35 +31,24 @@ import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCusto
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.GetActionCacheHelper;
import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.LogPair;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.commons.lang.NotImplementedException;
/******************************************************************************* /*******************************************************************************
@ -70,8 +57,6 @@ import org.apache.commons.lang.NotImplementedException;
*******************************************************************************/ *******************************************************************************/
public class GetAction public class GetAction
{ {
private static final QLogger LOG = QLogger.getLogger(InsertAction.class);
private Optional<AbstractPostQueryCustomizer> postGetRecordCustomizer; private Optional<AbstractPostQueryCustomizer> postGetRecordCustomizer;
private GetInput getInput; private GetInput getInput;
@ -125,36 +110,7 @@ public class GetAction
//////////////////////////// ////////////////////////////
if(table.getCacheOf() != null) if(table.getCacheOf() != null)
{ {
if(getOutput.getRecord() == null) new GetActionCacheHelper().handleCaching(getInput, getOutput);
{
///////////////////////////////////////////////////////////////////////
// if the record wasn't found, see if we should look in cache-source //
///////////////////////////////////////////////////////////////////////
QRecord recordFromSource = tryToGetFromCacheSource(getInput);
if(recordFromSource != null)
{
/////////////////////////////////////////////////////////////////////////////////////////////////
// good, we found a record from the source, make sure we should cache it, and if so, do it now //
/////////////////////////////////////////////////////////////////////////////////////////////////
QRecord recordToCache = mapSourceRecordToCacheRecord(table, recordFromSource);
boolean shouldCacheRecord = shouldCacheRecord(table, recordToCache);
if(shouldCacheRecord)
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(getInput.getTableName());
insertInput.setRecords(List.of(recordToCache));
InsertOutput insertOutput = new InsertAction().execute(insertInput);
getOutput.setRecord(insertOutput.getRecords().get(0));
}
}
}
else
{
/////////////////////////////////////////////////////////////////////////////////
// if the record was found, but it's too old, maybe re-fetch from cache source //
/////////////////////////////////////////////////////////////////////////////////
refreshCacheIfExpired(getInput, getOutput);
}
} }
//////////////////////////////////////////////////////// ////////////////////////////////////////////////////////
@ -170,173 +126,6 @@ public class GetAction
/*******************************************************************************
**
*******************************************************************************/
private boolean shouldCacheRecord(QTableMetaData table, QRecord recordToCache)
{
boolean shouldCacheRecord = true;
recordMatchExclusionLoop:
for(CacheUseCase useCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
{
for(QQueryFilter filter : CollectionUtils.nonNullList(useCase.getExcludeRecordsMatching()))
{
if(BackendQueryFilterUtils.doesRecordMatch(filter, recordToCache))
{
LOG.info("Not caching record because it matches a use case's filter exclusion", new LogPair("record", recordToCache), new LogPair("filter", filter));
shouldCacheRecord = false;
break recordMatchExclusionLoop;
}
}
}
return (shouldCacheRecord);
}
/*******************************************************************************
**
*******************************************************************************/
private static QRecord mapSourceRecordToCacheRecord(QTableMetaData table, QRecord recordFromSource)
{
QRecord cacheRecord = new QRecord(recordFromSource);
//////////////////////////////////////////////////////////////////////////////////////////////
// make sure every value in the qRecord is set, because we will possibly be doing an update //
// on this record and want to null out any fields not set, not leave them populated //
//////////////////////////////////////////////////////////////////////////////////////////////
for(String fieldName : table.getFields().keySet())
{
if(!cacheRecord.getValues().containsKey(fieldName))
{
cacheRecord.setValue(fieldName, null);
}
}
if(StringUtils.hasContent(table.getCacheOf().getCachedDateFieldName()))
{
cacheRecord.setValue(table.getCacheOf().getCachedDateFieldName(), Instant.now());
}
return (cacheRecord);
}
/*******************************************************************************
**
*******************************************************************************/
private void refreshCacheIfExpired(GetInput getInput, GetOutput getOutput) throws QException
{
QTableMetaData table = getInput.getTable();
Integer expirationSeconds = table.getCacheOf().getExpirationSeconds();
if(expirationSeconds != null)
{
QRecord cachedRecord = getOutput.getRecord();
Instant cachedDate = cachedRecord.getValueInstant(table.getCacheOf().getCachedDateFieldName());
if(cachedDate == null || cachedDate.isBefore(Instant.now().minus(expirationSeconds, ChronoUnit.SECONDS)))
{
//////////////////////////////////////////////////////////////////////////
// keep the serial key from the old record in case we need to delete it //
//////////////////////////////////////////////////////////////////////////
Serializable oldRecordPrimaryKey = getOutput.getRecord().getValue(table.getPrimaryKeyField());
boolean shouldDeleteCachedRecord = true;
///////////////////////////////////////////
// fetch record from original source now //
///////////////////////////////////////////
QRecord recordFromSource = tryToGetFromCacheSource(getInput);
if(recordFromSource != null)
{
//////////////////////////////////////////////////////////////////////
// if the record was found in the source, put it into the output //
// object so returned back to caller, check that it should actually //
// be cached before doing so //
//////////////////////////////////////////////////////////////////////
QRecord recordToCache = mapSourceRecordToCacheRecord(table, recordFromSource);
recordToCache.setValue(table.getPrimaryKeyField(), cachedRecord.getValue(table.getPrimaryKeyField()));
getOutput.setRecord(recordToCache);
if(shouldCacheRecord(table, recordToCache))
{
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(getInput.getTableName());
updateInput.setRecords(List.of(recordToCache));
UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
getOutput.setRecord(updateOutput.getRecords().get(0));
shouldDeleteCachedRecord = false;
}
}
else
{
///////////////////////////////////////////////////////////////////////////////////////
// if we did not get a record back from the source, empty out the getOutput's record //
///////////////////////////////////////////////////////////////////////////////////////
getOutput.setRecord(null);
}
if(shouldDeleteCachedRecord)
{
/////////////////////////////////////////////////////////////////////////////
// if the record is no longer in the source, then remove it from the cache //
/////////////////////////////////////////////////////////////////////////////
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(getInput.getTableName());
deleteInput.setPrimaryKeys(List.of(oldRecordPrimaryKey));
new DeleteAction().execute(deleteInput);
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private QRecord tryToGetFromCacheSource(GetInput getInput) throws QException
{
QRecord recordFromSource = null;
QTableMetaData table = getInput.getTable();
for(CacheUseCase cacheUseCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
{
if(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY.equals(cacheUseCase.getType()) && getInput.getUniqueKey() != null)
{
recordFromSource = getFromCachedSourceForUniqueKeyToUniqueKey(getInput, table.getCacheOf().getSourceTable());
break;
}
else
{
// todo!!
throw new NotImplementedException("Not-yet-implemented cache use case type: " + cacheUseCase.getType());
}
}
return (recordFromSource);
}
/*******************************************************************************
**
*******************************************************************************/
private QRecord getFromCachedSourceForUniqueKeyToUniqueKey(GetInput getInput, String sourceTableName) throws QException
{
/////////////////////////////////////////////////////
// do a Get on the source table, by the unique key //
/////////////////////////////////////////////////////
GetInput sourceGetInput = new GetInput();
sourceGetInput.setTableName(sourceTableName);
sourceGetInput.setUniqueKey(getInput.getUniqueKey());
GetOutput sourceGetOutput = new GetAction().execute(sourceGetInput);
QRecord outputRecord = sourceGetOutput.getRecord();
return (outputRecord);
}
/******************************************************************************* /*******************************************************************************
** Run a GetAction by using the QueryAction instead (e.g., with a filter made ** Run a GetAction by using the QueryAction instead (e.g., with a filter made
** from the pkey/ukey, and returning the single record if found). ** from the pkey/ukey, and returning the single record if found).
@ -356,42 +145,7 @@ public class GetAction
@Override @Override
public GetOutput execute(GetInput getInput) throws QException public GetOutput execute(GetInput getInput) throws QException
{ {
QueryInput queryInput = new QueryInput(); QueryInput queryInput = convertGetInputToQueryInput(getInput);
queryInput.setTableName(getInput.getTableName());
//////////////////////////////////////////////////
// build filter using either pkey or unique key //
//////////////////////////////////////////////////
QQueryFilter filter = new QQueryFilter();
if(getInput.getPrimaryKey() != null)
{
filter.addCriteria(new QFilterCriteria(getInput.getTable().getPrimaryKeyField(), QCriteriaOperator.EQUALS, getInput.getPrimaryKey()));
}
else if(getInput.getUniqueKey() != null)
{
for(Map.Entry<String, Serializable> entry : getInput.getUniqueKey().entrySet())
{
if(entry.getValue() == null)
{
filter.addCriteria(new QFilterCriteria(entry.getKey(), QCriteriaOperator.IS_BLANK));
}
else
{
filter.addCriteria(new QFilterCriteria(entry.getKey(), QCriteriaOperator.EQUALS, entry.getValue()));
}
}
}
else
{
throw (new QException("No primaryKey or uniqueKey was passed to Get"));
}
queryInput.setFilter(filter);
queryInput.setIncludeAssociations(getInput.getIncludeAssociations());
queryInput.setAssociationNamesToInclude(getInput.getAssociationNamesToInclude());
queryInput.setShouldFetchHeavyFields(getInput.getShouldFetchHeavyFields());
queryInput.setShouldMaskPasswords(getInput.getShouldMaskPasswords());
queryInput.setShouldOmitHiddenFields(getInput.getShouldOmitHiddenFields());
QueryOutput queryOutput = new QueryAction().execute(queryInput); QueryOutput queryOutput = new QueryAction().execute(queryInput);
@ -406,6 +160,55 @@ public class GetAction
/*******************************************************************************
**
*******************************************************************************/
public static QueryInput convertGetInputToQueryInput(GetInput getInput) throws QException
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(getInput.getTableName());
//////////////////////////////////////////////////
// build filter using either pkey or unique key //
//////////////////////////////////////////////////
QQueryFilter filter = new QQueryFilter();
if(getInput.getPrimaryKey() != null)
{
filter.addCriteria(new QFilterCriteria(getInput.getTable().getPrimaryKeyField(), QCriteriaOperator.EQUALS, getInput.getPrimaryKey()));
}
else if(getInput.getUniqueKey() != null)
{
for(Map.Entry<String, Serializable> entry : getInput.getUniqueKey().entrySet())
{
if(entry.getValue() == null)
{
filter.addCriteria(new QFilterCriteria(entry.getKey(), QCriteriaOperator.IS_BLANK));
}
else
{
filter.addCriteria(new QFilterCriteria(entry.getKey(), QCriteriaOperator.EQUALS, entry.getValue()));
}
}
}
else
{
throw (new QException("No primaryKey or uniqueKey was passed to Get"));
}
queryInput.setFilter(filter);
queryInput.setTransaction(getInput.getTransaction());
queryInput.setIncludeAssociations(getInput.getIncludeAssociations());
queryInput.setAssociationNamesToInclude(getInput.getAssociationNamesToInclude());
queryInput.setShouldTranslatePossibleValues(getInput.getShouldTranslatePossibleValues());
queryInput.setShouldGenerateDisplayValues(getInput.getShouldGenerateDisplayValues());
queryInput.setShouldFetchHeavyFields(getInput.getShouldFetchHeavyFields());
queryInput.setShouldMaskPasswords(getInput.getShouldMaskPasswords());
queryInput.setShouldOmitHiddenFields(getInput.getShouldOmitHiddenFields());
return queryInput;
}
/******************************************************************************* /*******************************************************************************
** Run the necessary actions on a record. This may include setting display values, ** Run the necessary actions on a record. This may include setting display values,
** translating possible values, and running post-record customizations. ** translating possible values, and running post-record customizations.

14
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java
View File

@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe; import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe;
import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipeBufferedWrapper; import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipeBufferedWrapper;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryActionCacheHelper;
import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.context.QContext;
@ -87,12 +88,13 @@ public class QueryAction
throw (new QException("Table name was not specified in query input")); throw (new QException("Table name was not specified in query input"));
} }
if(queryInput.getTable() == null) QTableMetaData table = queryInput.getTable();
if(table == null)
{ {
throw (new QException("A table named [" + queryInput.getTableName() + "] was not found in the active QInstance")); throw (new QException("A table named [" + queryInput.getTableName() + "] was not found in the active QInstance"));
} }
postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, queryInput.getTable(), TableCustomizers.POST_QUERY_RECORD.getRole()); postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole());
this.queryInput = queryInput; this.queryInput = queryInput;
if(queryInput.getRecordPipe() != null) if(queryInput.getRecordPipe() != null)
@ -115,6 +117,14 @@ public class QueryAction
QueryOutput queryOutput = qModule.getQueryInterface().execute(queryInput); QueryOutput queryOutput = qModule.getQueryInterface().execute(queryInput);
// todo post-customization - can do whatever w/ the result if you want // todo post-customization - can do whatever w/ the result if you want
////////////////////////////
// handle cache use-cases //
////////////////////////////
if(table.getCacheOf() != null)
{
new QueryActionCacheHelper().handleCaching(queryInput, queryOutput);
}
if(queryInput.getRecordPipe() instanceof BufferedRecordPipe bufferedRecordPipe) if(queryInput.getRecordPipe() instanceof BufferedRecordPipe bufferedRecordPipe)
{ {
bufferedRecordPipe.finalFlush(); bufferedRecordPipe.finalFlush();

101
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/CacheUtils.java Normal file
View File

@ -0,0 +1,101 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.tables.helpers;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.logging.LogPair;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
**
*******************************************************************************/
public class CacheUtils
{
private static final QLogger LOG = QLogger.getLogger(CacheUtils.class);
/*******************************************************************************
**
*******************************************************************************/
static QRecord mapSourceRecordToCacheRecord(QTableMetaData table, QRecord recordFromSource)
{
QRecord cacheRecord = new QRecord(recordFromSource);
//////////////////////////////////////////////////////////////////////////////////////////////
// make sure every value in the qRecord is set, because we will possibly be doing an update //
// on this record and want to null out any fields not set, not leave them populated //
//////////////////////////////////////////////////////////////////////////////////////////////
for(String fieldName : table.getFields().keySet())
{
if(fieldName.equals(table.getPrimaryKeyField()))
{
cacheRecord.removeValue(fieldName);
}
else if(!cacheRecord.getValues().containsKey(fieldName))
{
cacheRecord.setValue(fieldName, null);
}
}
if(StringUtils.hasContent(table.getCacheOf().getCachedDateFieldName()))
{
cacheRecord.setValue(table.getCacheOf().getCachedDateFieldName(), Instant.now());
}
return (cacheRecord);
}
/*******************************************************************************
**
*******************************************************************************/
static boolean shouldCacheRecord(QTableMetaData table, QRecord recordToCache)
{
boolean shouldCacheRecord = true;
recordMatchExclusionLoop:
for(CacheUseCase useCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
{
for(QQueryFilter filter : CollectionUtils.nonNullList(useCase.getExcludeRecordsMatching()))
{
if(BackendQueryFilterUtils.doesRecordMatch(filter, recordToCache))
{
LOG.info("Not caching record because it matches a use case's filter exclusion", new LogPair("record", recordToCache), new LogPair("filter", filter));
shouldCacheRecord = false;
break recordMatchExclusionLoop;
}
}
}
return (shouldCacheRecord);
}
}

241
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/GetActionCacheHelper.java Normal file
View File

@ -0,0 +1,241 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.tables.helpers;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
/*******************************************************************************
**
*******************************************************************************/
public class GetActionCacheHelper
{
private static final QLogger LOG = QLogger.getLogger(GetActionCacheHelper.class);
/*******************************************************************************
**
*******************************************************************************/
public void handleCaching(GetInput getInput, GetOutput getOutput) throws QException
{
///////////////////////////////////////////////////////
// copy Get input & output into Query input & output //
///////////////////////////////////////////////////////
QueryInput queryInput = GetAction.convertGetInputToQueryInput(getInput);
QueryOutput queryOutput = new QueryOutput(queryInput);
if(getOutput.getRecord() != null)
{
queryOutput.addRecord(getOutput.getRecord());
}
////////////////////////////////////
// run the QueryActionCacheHelper //
////////////////////////////////////
new QueryActionCacheHelper().handleCaching(queryInput, queryOutput);
///////////////////////////////////
// set result back in get output //
///////////////////////////////////
if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
{
getOutput.setRecord(queryOutput.getRecords().get(0));
}
else
{
getOutput.setRecord(null);
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// In July 2023, initial caching was added in QueryAction. //
// at this time, it felt wrong to essentially duplicate this code between Get & Query - as Get is a simplified use-case of Query. //
// so - we'll keep this code here, as a potential quick/easy fallback - but - see above - where we use QueryActionCacheHelper instead. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/*
public void handleCaching(GetInput getInput, GetOutput getOutput) throws QException
{
if(getOutput.getRecord() == null)
{
///////////////////////////////////////////////////////////////////////
// if the record wasn't found, see if we should look in cache-source //
///////////////////////////////////////////////////////////////////////
QRecord recordFromSource = tryToGetFromCacheSource(getInput);
if(recordFromSource != null)
{
/////////////////////////////////////////////////////////////////////////////////////////////////
// good, we found a record from the source, make sure we should cache it, and if so, do it now //
// note, we always return the record from the source, even if we don't cache it. //
/////////////////////////////////////////////////////////////////////////////////////////////////
QTableMetaData table = getInput.getTable();
QRecord recordToCache = CacheUtils.mapSourceRecordToCacheRecord(table, recordFromSource);
getOutput.setRecord(recordToCache);
boolean shouldCacheRecord = CacheUtils.shouldCacheRecord(table, recordToCache);
if(shouldCacheRecord)
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(getInput.getTableName());
insertInput.setRecords(List.of(recordToCache));
InsertOutput insertOutput = new InsertAction().execute(insertInput);
/////////////////////////////////////////////////////////////////////////////////////////////
// update the result record from the insert (e.g., so we get its id, just in case we care) //
/////////////////////////////////////////////////////////////////////////////////////////////
getOutput.setRecord(insertOutput.getRecords().get(0));
}
}
}
else
{
/////////////////////////////////////////////////////////////////////////////////
// if the record was found, but it's too old, maybe re-fetch from cache source //
/////////////////////////////////////////////////////////////////////////////////
refreshCacheIfExpired(getInput, getOutput);
}
}
private QRecord tryToGetFromCacheSource(GetInput getInput) throws QException
{
QRecord recordFromSource = null;
QTableMetaData table = getInput.getTable();
for(CacheUseCase cacheUseCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
{
if(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY.equals(cacheUseCase.getType()) && getInput.getUniqueKey() != null)
{
recordFromSource = getFromCachedSourceForUniqueKeyToUniqueKey(getInput, table.getCacheOf().getSourceTable());
break;
}
else
{
// todo!!
throw new NotImplementedException("Not-yet-implemented cache use case type: " + cacheUseCase.getType());
}
}
return (recordFromSource);
}
private void refreshCacheIfExpired(GetInput getInput, GetOutput getOutput) throws QException
{
QTableMetaData table = getInput.getTable();
Integer expirationSeconds = table.getCacheOf().getExpirationSeconds();
if(expirationSeconds != null)
{
QRecord cachedRecord = getOutput.getRecord();
Instant cachedDate = cachedRecord.getValueInstant(table.getCacheOf().getCachedDateFieldName());
if(cachedDate == null || cachedDate.isBefore(Instant.now().minus(expirationSeconds, ChronoUnit.SECONDS)))
{
//////////////////////////////////////////////////////////////////////////
// keep the serial key from the old record in case we need to delete it //
//////////////////////////////////////////////////////////////////////////
Serializable oldRecordPrimaryKey = cachedRecord.getValue(table.getPrimaryKeyField());
boolean shouldDeleteCachedRecord;
///////////////////////////////////////////
// fetch record from original source now //
///////////////////////////////////////////
QRecord recordFromSource = tryToGetFromCacheSource(getInput);
if(recordFromSource != null)
{
///////////////////////////////////////////////////////////////////
// if the record was found in the source, put it into the output //
// object so returned back to caller //
///////////////////////////////////////////////////////////////////
QRecord recordToCache = CacheUtils.mapSourceRecordToCacheRecord(table, recordFromSource);
recordToCache.setValue(table.getPrimaryKeyField(), cachedRecord.getValue(table.getPrimaryKeyField()));
getOutput.setRecord(recordToCache);
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the record should be cached, update the cache record - else set the flag to delete the cached record. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(CacheUtils.shouldCacheRecord(table, recordToCache))
{
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(getInput.getTableName());
updateInput.setRecords(List.of(recordToCache));
UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
getOutput.setRecord(updateOutput.getRecords().get(0));
shouldDeleteCachedRecord = false;
}
else
{
shouldDeleteCachedRecord = true;
}
}
else
{
///////////////////////////////////////////////////////////////////////////////////////
// if we did not get a record back from the source, empty out the getOutput's record //
// and set the flag to delete the cached record //
///////////////////////////////////////////////////////////////////////////////////////
getOutput.setRecord(null);
shouldDeleteCachedRecord = true;
}
if(shouldDeleteCachedRecord)
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the record is no longer in the source (or it was in the source, but failed the should-cache check), then remove it from the cache //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(getInput.getTableName());
deleteInput.setPrimaryKeys(List.of(oldRecordPrimaryKey));
new DeleteAction().execute(deleteInput);
}
}
}
}
private QRecord getFromCachedSourceForUniqueKeyToUniqueKey(GetInput getInput, String sourceTableName) throws QException
{
/////////////////////////////////////////////////////
// do a Get on the source table, by the unique key //
/////////////////////////////////////////////////////
GetInput sourceGetInput = new GetInput();
sourceGetInput.setTableName(sourceTableName);
sourceGetInput.setUniqueKey(getInput.getUniqueKey());
GetOutput sourceGetOutput = new GetAction().execute(sourceGetInput);
QRecord outputRecord = sourceGetOutput.getRecord();
return (outputRecord);
}
*/
}

527
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryActionCacheHelper.java Normal file
View File

@ -0,0 +1,527 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.tables.helpers;
import java.io.Serializable;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ListingHash;
import org.apache.commons.lang.NotImplementedException;
/*******************************************************************************
** After running a query, if it's for a table that's a CacheOf another table,
** see if there are any cache use-cases to apply to the query result.
**
** Such as:
** - if it's a query for one or more values in a UniqueKey:
** - if any particular UniqueKeys weren't found, look in the source table
** - if any cached records are expired, refresh them from the source
** - possibly updating the cached record; possibly deleting it.
*******************************************************************************/
public class QueryActionCacheHelper
{
private static final QLogger LOG = QLogger.getLogger(QueryActionCacheHelper.class);
private boolean isQueryInputCacheable = false;
private Set<CacheUseCase.Type> cacheUseCases = new HashSet<>();
private CacheUseCase.Type activeCacheUseCase = null;
private UniqueKey cacheUniqueKey = null;
private ListingHash<String, Serializable> uniqueKeyValues = new ListingHash<>();
/*******************************************************************************
**
*******************************************************************************/
public void handleCaching(QueryInput queryInput, QueryOutput queryOutput) throws QException
{
analyzeInput(queryInput);
if(!isQueryInputCacheable)
{
return;
}
//////////////////////////////////////////////////////////////////////////
// figure out which keys in the query were found, and which were missed //
//////////////////////////////////////////////////////////////////////////
List<QRecord> recordsFoundInCache = new ArrayList<>(queryOutput.getRecords());
Set<List<Serializable>> uniqueKeyValuesInFoundRecords = getUniqueKeyValuesFromFoundRecords(queryOutput.getRecords());
Set<List<Serializable>> missedUniqueKeyValues = getUniqueKeyValuesFromQuery();
missedUniqueKeyValues.removeAll(uniqueKeyValuesInFoundRecords);
///////////////////////////////////////////////////////////////////////////////////
// if any requested records weren't found, see if we should look in cache-source //
///////////////////////////////////////////////////////////////////////////////////
if(CollectionUtils.nullSafeHasContents(missedUniqueKeyValues))
{
List<QRecord> recordsFromSource = tryToGetFromCacheSource(queryInput, missedUniqueKeyValues);
if(CollectionUtils.nullSafeHasContents(recordsFromSource))
{
//////////////////////////////////////////////////////////////////////////////////////////////////
// good, we found records from the source, make sure we should cache them, and if so, do it now //
// note, we always return the record from the source, even if we don't cache it. //
//////////////////////////////////////////////////////////////////////////////////////////////////
QTableMetaData table = queryInput.getTable();
List<QRecord> recordsToReturn = recordsFromSource.stream()
.map(r -> CacheUtils.mapSourceRecordToCacheRecord(table, r))
.toList();
queryOutput.addRecords(recordsToReturn);
List<QRecord> recordsToCache = recordsToReturn.stream()
.filter(r -> CacheUtils.shouldCacheRecord(table, r))
.toList();
if(CollectionUtils.nullSafeHasContents(recordsToCache))
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(queryInput.getTableName());
insertInput.setRecords(recordsToCache);
insertInput.setSkipUniqueKeyCheck(true);
InsertOutput insertOutput = new InsertAction().execute(insertInput);
}
}
}
//////////////////////////////////////////////////////////////////////////
// for records that were found, if they're too old, maybe re-fetch them //
//////////////////////////////////////////////////////////////////////////
if(CollectionUtils.nullSafeHasContents(recordsFoundInCache))
{
refreshCacheIfExpired(recordsFoundInCache, queryInput, queryOutput);
}
}
/*******************************************************************************
**
*******************************************************************************/
private void refreshCacheIfExpired(List<QRecord> recordsFoundInCache, QueryInput queryInput, QueryOutput queryOutput) throws QException
{
QTableMetaData table = queryInput.getTable();
Integer expirationSeconds = table.getCacheOf().getExpirationSeconds();
if(expirationSeconds != null)
{
List<QRecord> expiredRecords = new ArrayList<>();
for(QRecord cachedRecord : recordsFoundInCache)
{
Instant cachedDate = cachedRecord.getValueInstant(table.getCacheOf().getCachedDateFieldName());
if(cachedDate == null || cachedDate.isBefore(Instant.now().minus(expirationSeconds, ChronoUnit.SECONDS)))
{
expiredRecords.add(cachedRecord);
}
}
if(CollectionUtils.nullSafeHasContents(expiredRecords))
{
Map<List<Serializable>, Serializable> uniqueKeyToPrimaryKeyMap = getUniqueKeyToPrimaryKeyMap(table.getPrimaryKeyField(), expiredRecords);
Set<List<Serializable>> uniqueKeyValuesToRefresh = uniqueKeyToPrimaryKeyMap.keySet();
////////////////////////////////////////////
// fetch records from original source now //
////////////////////////////////////////////
List<QRecord> recordsFromSource = tryToGetFromCacheSource(queryInput, uniqueKeyValuesToRefresh);
Set<List<Serializable>> uniqueKeyValuesInFoundRecords = getUniqueKeyValuesFromFoundRecords(recordsFromSource);
Set<List<Serializable>> missedUniqueKeyValues = getUniqueKeyValuesFromQuery();
missedUniqueKeyValues.retainAll(getUniqueKeyValuesFromFoundRecords(expiredRecords));
missedUniqueKeyValues.removeAll(uniqueKeyValuesInFoundRecords);
//////////////////////////////////////////////////////////////////////////////////////////////////////
// build records to cache - setting their original (from cache) ids back in them, so they'll update //
//////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> refreshedRecordsToReturn = recordsFromSource.stream()
.map(r ->
{
QRecord recordToCache = CacheUtils.mapSourceRecordToCacheRecord(table, r);
recordToCache.setValue(table.getPrimaryKeyField(), uniqueKeyToPrimaryKeyMap.get(getUniqueKeyValues(recordToCache)));
return (recordToCache);
})
.toList();
///////////////////////////////////////////////////////////////////////////////////////////////////////
// if the records were found in the source, put it into the output object so returned back to caller //
///////////////////////////////////////////////////////////////////////////////////////////////////////
Map<List<Serializable>, QRecord> refreshedRecordsByUniqueKeyValues = refreshedRecordsToReturn.stream().collect(Collectors.toMap(this::getUniqueKeyValues, r -> r, (a, b) -> a));
ListIterator<QRecord> queryOutputListIterator = queryOutput.getRecords().listIterator();
while(queryOutputListIterator.hasNext())
{
QRecord originalRecord = queryOutputListIterator.next();
List<Serializable> recordUniqueKeyValues = getUniqueKeyValues(originalRecord);
QRecord refreshedRecord = refreshedRecordsByUniqueKeyValues.get(recordUniqueKeyValues);
if(refreshedRecord != null)
{
queryOutputListIterator.set(refreshedRecord);
}
else if(missedUniqueKeyValues.contains(recordUniqueKeyValues))
{
queryOutputListIterator.remove();
}
}
////////////////////////////////////////////////////////////////////////////
// for refreshed records which should be cached, update them in the cache //
////////////////////////////////////////////////////////////////////////////
List<QRecord> recordsToUpdate = refreshedRecordsToReturn.stream().filter(r -> CacheUtils.shouldCacheRecord(table, r)).toList();
if(CollectionUtils.nullSafeHasContents(recordsToUpdate))
{
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(queryInput.getTableName());
updateInput.setRecords(recordsToUpdate);
UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the records were missed in the source - OR if they shouldn't be cached now, then mark them for deleting //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Set<Serializable> cachedRecordIdsToDelete = new HashSet<>(missedUniqueKeyValues.stream()
.map(ukValues -> uniqueKeyToPrimaryKeyMap.get(ukValues)).collect(Collectors.toSet()));
cachedRecordIdsToDelete.addAll(refreshedRecordsToReturn.stream()
.filter(r -> !CacheUtils.shouldCacheRecord(table, r))
.map(r -> r.getValue(table.getPrimaryKeyField()))
.collect(Collectors.toSet()));
if(CollectionUtils.nullSafeHasContents(cachedRecordIdsToDelete))
{
/////////////////////////////////////////////////////////////////////////////////
// if the records are no longer in the source, then remove them from the cache //
/////////////////////////////////////////////////////////////////////////////////
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(queryInput.getTableName());
deleteInput.setPrimaryKeys(new ArrayList<>(cachedRecordIdsToDelete));
new DeleteAction().execute(deleteInput);
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private Set<List<Serializable>> getUniqueKeyValuesFromQuery()
{
Set<List<Serializable>> rs = new HashSet<>();
int noOfUniqueKeys = uniqueKeyValues.get(cacheUniqueKey.getFieldNames().get(0)).size();
for(int i = 0; i < noOfUniqueKeys; i++)
{
List<Serializable> values = new ArrayList<>();
for(String fieldName : cacheUniqueKey.getFieldNames())
{
values.add(uniqueKeyValues.get(fieldName).get(i));
}
////////////////////////////////////////////////////////////////////////////////
// critical - leave this here so hashCode from the list is correctly computed //
////////////////////////////////////////////////////////////////////////////////
rs.add(values);
}
return (rs);
}
/*******************************************************************************
**
*******************************************************************************/
private Set<List<Serializable>> getUniqueKeyValuesFromFoundRecords(List<QRecord> records)
{
return (getUniqueKeyToPrimaryKeyMap("ignore", records).keySet());
}
/*******************************************************************************
**
*******************************************************************************/
private Map<List<Serializable>, Serializable> getUniqueKeyToPrimaryKeyMap(String primaryKeyField, List<QRecord> records)
{
Map<List<Serializable>, Serializable> rs = new HashMap<>();
for(QRecord record : records)
{
List<Serializable> uniqueKeyValues = getUniqueKeyValues(record);
rs.put(uniqueKeyValues, record.getValue(primaryKeyField));
}
return (rs);
}
/*******************************************************************************
**
*******************************************************************************/
private List<Serializable> getUniqueKeyValues(QRecord record)
{
List<Serializable> uniqueKeyValues = new ArrayList<>();
for(String fieldName : cacheUniqueKey.getFieldNames())
{
uniqueKeyValues.add(record.getValue(fieldName));
}
return uniqueKeyValues;
}
/*******************************************************************************
** figure out if this was a request that we can cache records for -
** e.g., if it's a request for unique-key EQUALS or IN
** build up fields for the unique keys, the values, etc.
*******************************************************************************/
private void analyzeInput(QueryInput queryInput)
{
QTableMetaData table = queryInput.getTable();
for(CacheUseCase cacheUseCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
{
cacheUseCases.add(cacheUseCase.getType());
}
if(cacheUseCases.contains(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY))
{
if(queryInput.getFilter() == null)
{
LOG.trace("Unable to cache: there is no filter");
return;
}
QQueryFilter filter = queryInput.getFilter();
Set<String> queryFields = new HashSet<>();
if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
{
if(CollectionUtils.nullSafeHasContents(filter.getCriteria()))
{
LOG.trace("Unable to cache: we have sub-filters and criteria");
return;
}
if(!QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
{
LOG.trace("Unable to cache: we have sub-filters but not an OR query");
return;
}
/////////////////////////
// look at sub-filters //
/////////////////////////
for(QQueryFilter subFilter : filter.getSubFilters())
{
Set<String> thisSubFilterFields = getQueryFieldsIfCacheableFilter(subFilter, false);
if(thisSubFilterFields == null)
{
return;
}
if(queryFields.isEmpty())
{
queryFields.addAll(thisSubFilterFields);
}
else
{
if(!queryFields.equals(thisSubFilterFields))
{
LOG.trace("Unable to cache: sub-filters have different sets of fields");
return;
}
}
}
if(doQueryFieldsMatchAUniqueKey(table, queryFields))
{
return;
}
LOG.trace("Unable to cache: we have sub-filters that do match a unique key");
return;
}
else
{
//////////////////////////////////////////
// look at the criteria in the query: //
// - build a set of field names //
// - fail upon unsupported operators //
// - collect the values in the criteria //
//////////////////////////////////////////
queryFields = getQueryFieldsIfCacheableFilter(filter, true);
if(queryFields == null)
{
return;
}
}
if(doQueryFieldsMatchAUniqueKey(table, queryFields))
{
return;
}
LOG.trace("Unable to cache: we have query fields that don't match a unique key: " + queryFields);
return;
}
LOG.trace("Unable to cache: No supported use case: " + cacheUseCases);
}
/*******************************************************************************
**
*******************************************************************************/
private boolean doQueryFieldsMatchAUniqueKey(QTableMetaData table, Set<String> queryFields)
{
for(UniqueKey uniqueKey : CollectionUtils.nonNullList(table.getUniqueKeys()))
{
if(queryFields.equals(new HashSet<>(uniqueKey.getFieldNames())))
{
this.cacheUniqueKey = uniqueKey;
isQueryInputCacheable = true;
activeCacheUseCase = CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY;
return true;
}
}
return false;
}
/*******************************************************************************
**
*******************************************************************************/
private Set<String> getQueryFieldsIfCacheableFilter(QQueryFilter filter, boolean allowOperatorIn)
{
Set<String> rs = new HashSet<>();
for(QFilterCriteria criterion : filter.getCriteria())
{
boolean isEquals = criterion.getOperator().equals(QCriteriaOperator.EQUALS);
boolean isIn = criterion.getOperator().equals(QCriteriaOperator.IN);
if(isEquals || (isIn && allowOperatorIn))
{
rs.add(criterion.getFieldName());
this.uniqueKeyValues.addAll(criterion.getFieldName(), criterion.getValues());
}
else
{
LOG.trace("Unable to cache: we have an unsupported criteria operator: " + criterion.getOperator());
isQueryInputCacheable = false;
return (null);
}
}
return (rs);
}
/*******************************************************************************
**
*******************************************************************************/
private List<QRecord> tryToGetFromCacheSource(QueryInput queryInput, Set<List<Serializable>> uniqueKeyValues) throws QException
{
List<QRecord> recordsFromSource = null;
QTableMetaData table = queryInput.getTable();
if(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY.equals(activeCacheUseCase))
{
recordsFromSource = getFromCachedSourceForUniqueKeyToUniqueKey(uniqueKeyValues, table.getCacheOf().getSourceTable());
}
else
{
// todo!!
throw (new NotImplementedException("Not-yet-implemented cache use case type: " + activeCacheUseCase));
}
return (recordsFromSource);
}
/*******************************************************************************
**
*******************************************************************************/
private List<QRecord> getFromCachedSourceForUniqueKeyToUniqueKey(Set<List<Serializable>> uniqueKeyValues, String sourceTableName) throws QException
{
///////////////////////////////////////////////////////
// do a Query on the source table, by the unique key //
///////////////////////////////////////////////////////
QueryInput sourceQueryInput = new QueryInput();
sourceQueryInput.setTableName(sourceTableName);
QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
sourceQueryInput.setFilter(filter);
for(List<Serializable> uniqueKeyValue : uniqueKeyValues)
{
QQueryFilter subFilter = new QQueryFilter();
filter.addSubFilter(subFilter);
for(int i = 0; i < cacheUniqueKey.getFieldNames().size(); i++)
{
subFilter.addCriteria(new QFilterCriteria(cacheUniqueKey.getFieldNames().get(i), QCriteriaOperator.EQUALS, uniqueKeyValue.get(i)));
}
}
QueryOutput sourceQueryOutput = new QueryAction().execute(sourceQueryInput);
return (sourceQueryOutput.getRecords());
}
}

176
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/GetActionTest.java
View File

@ -22,29 +22,15 @@
package com.kingsrook.qqq.backend.core.actions.tables; package com.kingsrook.qqq.backend.core.actions.tables;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
/******************************************************************************* /*******************************************************************************
@ -85,166 +71,4 @@ class GetActionTest extends BaseTest
assertNotNull(result.getRecord()); assertNotNull(result.getRecord());
} }
/*******************************************************************************
**
*******************************************************************************/
@Test
void testUniqueKeyCache() throws QException
{
QInstance qInstance = QContext.getQInstance();
/////////////////////////////////////
// insert rows in the source table //
/////////////////////////////////////
TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of(
new QRecord().withValue("id", 1).withValue("firstName", "George").withValue("lastName", "Washington").withValue("noOfShoes", 5),
new QRecord().withValue("id", 2).withValue("firstName", "John").withValue("lastName", "Adams"),
new QRecord().withValue("id", 3).withValue("firstName", "Thomas").withValue("lastName", "Jefferson"),
new QRecord().withValue("id", 4).withValue("firstName", "Thomas 503").withValue("lastName", "Jefferson"),
new QRecord().withValue("id", 5).withValue("firstName", "Thomas 999").withValue("lastName", "Jefferson")
));
/////////////////////////////////////////////////////////////////////////////
// get from the table which caches it - confirm they are (magically) found //
/////////////////////////////////////////////////////////////////////////////
{
GetInput getInput = new GetInput();
getInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE);
getInput.setUniqueKey(Map.of("firstName", "George", "lastName", "Washington"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
assertNotNull(getOutput.getRecord().getValue("cachedDate"));
assertEquals(5, getOutput.getRecord().getValue("noOfShoes"));
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// try to get from the table which caches it - but should not find because use case should filter out because of matching 503 //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
{
GetInput getInput = new GetInput();
getInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE);
getInput.setUniqueKey(Map.of("firstName", "Thomas 503", "lastName", "Jefferson"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNull(getOutput.getRecord());
getInput.setUniqueKey(Map.of("firstName", "Thomas 999", "lastName", "Jefferson"));
getOutput = new GetAction().execute(getInput);
assertNull(getOutput.getRecord());
}
///////////////////////////////////////////////////////////////////////////
// request a row that doesn't exist in cache or source, should miss both //
///////////////////////////////////////////////////////////////////////////
{
GetInput getInput = new GetInput();
getInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE);
getInput.setUniqueKey(Map.of("firstName", "John", "lastName", "McCain"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNull(getOutput.getRecord());
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// update the record in the source table - then re-get from cache table - shouldn't see new value. //
/////////////////////////////////////////////////////////////////////////////////////////////////////
{
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
updateInput.setRecords(List.of(new QRecord().withValue("id", 1).withValue("noOfShoes", 6)));
new UpdateAction().execute(updateInput);
GetInput getInput = new GetInput();
getInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE);
getInput.setUniqueKey(Map.of("firstName", "George", "lastName", "Washington"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
assertNotNull(getOutput.getRecord().getValue("cachedDate"));
assertEquals(5, getOutput.getRecord().getValue("noOfShoes"));
}
///////////////////////////////////////////////////////////////////////////
// delete the cached record; re-get, and we should see the updated value //
///////////////////////////////////////////////////////////////////////////
{
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE);
deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, "George")));
new DeleteAction().execute(deleteInput);
GetInput getInput = new GetInput();
getInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE);
getInput.setUniqueKey(Map.of("firstName", "George", "lastName", "Washington"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
assertNotNull(getOutput.getRecord().getValue("cachedDate"));
assertEquals(6, getOutput.getRecord().getValue("noOfShoes"));
}
///////////////////////////////////////////////////////////////////
// update the source record; see that it isn't updated in cache. //
///////////////////////////////////////////////////////////////////
{
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
updateInput.setRecords(List.of(new QRecord().withValue("id", 1).withValue("noOfShoes", 7)));
new UpdateAction().execute(updateInput);
GetInput getInput = new GetInput();
getInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE);
getInput.setUniqueKey(Map.of("firstName", "George", "lastName", "Washington"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
assertNotNull(getOutput.getRecord().getValue("cachedDate"));
assertEquals(6, getOutput.getRecord().getValue("noOfShoes"));
///////////////////////////////////////////////////////////////////////
// then artificially move back the cachedDate in the cache table. //
// then re-get from cache table, and we should see the updated value //
///////////////////////////////////////////////////////////////////////
updateInput = new UpdateInput();
updateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE);
updateInput.setRecords(List.of(getOutput.getRecord().withValue("cachedDate", Instant.parse("2001-01-01T00:00:00Z"))));
new UpdateAction().execute(updateInput);
getOutput = new GetAction().execute(getInput);
assertEquals(7, getOutput.getRecord().getValue("noOfShoes"));
}
/////////////////////////////////////////////////
// should only be 1 cache record at this point //
/////////////////////////////////////////////////
assertEquals(1, TestUtils.queryTable(QContext.getQInstance(), TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE).size());
//////////////////////////////////////////////////////////////////////
// delete the source record - it will still be in the cache though. //
//////////////////////////////////////////////////////////////////////
{
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
deleteInput.setPrimaryKeys(List.of(1));
new DeleteAction().execute(deleteInput);
GetInput getInput = new GetInput();
getInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE);
getInput.setUniqueKey(Map.of("firstName", "George", "lastName", "Washington"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
////////////////////////////////////////////////////////////////////
// then artificially move back the cachedDate in the cache table. //
// then re-get from cache table, and now it should go away //
////////////////////////////////////////////////////////////////////
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE);
updateInput.setRecords(List.of(getOutput.getRecord().withValue("cachedDate", Instant.parse("2001-01-01T00:00:00Z"))));
new UpdateAction().execute(updateInput);
getInput = new GetInput();
getInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE);
getInput.setUniqueKey(Map.of("firstName", "George", "lastName", "Washington"));
getOutput = new GetAction().execute(getInput);
assertNull(getOutput.getRecord());
}
}
} }

276
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/GetActionCacheHelperTest.java Normal file
View File

@ -0,0 +1,276 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.tables.helpers;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for GetActionCacheHelper
*******************************************************************************/
class GetActionCacheHelperTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testUniqueKeyCache() throws QException
{
QInstance qInstance = QContext.getQInstance();
String sourceTableName = TestUtils.TABLE_NAME_PERSON_MEMORY;
String cacheTableName = TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE;
/////////////////////////////////////
// insert rows in the source table //
/////////////////////////////////////
TestUtils.insertRecords(qInstance, qInstance.getTable(sourceTableName), List.of(
new QRecord().withValue("id", 1).withValue("firstName", "George").withValue("lastName", "Washington").withValue("noOfShoes", 5),
new QRecord().withValue("id", 2).withValue("firstName", "John").withValue("lastName", "Adams"),
new QRecord().withValue("id", 3).withValue("firstName", "Thomas").withValue("lastName", "Jefferson"),
new QRecord().withValue("id", 4).withValue("firstName", "James").withValue("lastName", "Garfield").withValue("noOfShoes", 503),
new QRecord().withValue("id", 5).withValue("firstName", "Abraham").withValue("lastName", "Lincoln").withValue("noOfShoes", 999),
new QRecord().withValue("id", 6).withValue("firstName", "Bill").withValue("lastName", "Clinton")
));
/////////////////////////////////////////////////////////////////////////////
// get from the table which caches it - confirm they are (magically) found //
/////////////////////////////////////////////////////////////////////////////
{
GetInput getInput = new GetInput();
getInput.setTableName(cacheTableName);
getInput.setUniqueKey(Map.of("firstName", "George", "lastName", "Washington"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
assertNotNull(getOutput.getRecord().getValue("cachedDate"));
assertEquals(5, getOutput.getRecord().getValue("noOfShoes"));
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// try to get records through the cache table, which meet the conditions that cause them to not be cached. //
// so we should get results from the Get request - but - then let's go directly to the backend to confirm the records are not cached. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
{
GetInput getInput = new GetInput();
getInput.setTableName(cacheTableName);
getInput.setUniqueKey(Map.of("firstName", "James", "lastName", "Garfield"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
getInput.setUniqueKey(Map.of("firstName", "Abraham", "lastName", "Lincoln"));
getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
assertEquals(0, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter(new QFilterCriteria("firstName", QCriteriaOperator.IN, "Abraham", "James"))));
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// fetch a record through the cache, so it gets cached. //
// then update the source record so that it meets the condition that doesn't allow it to be cached. //
// then expire the cached record. //
// then re-fetch through cache - which should see the expiration, re-fetch from source, and delete from cache. //
// assert record is no longer in cache. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
{
GetInput getInput = new GetInput();
getInput.setTableName(cacheTableName);
getInput.setUniqueKey(Map.of("firstName", "Bill", "lastName", "Clinton"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
assertNotNull(getOutput.getRecord().getValue("cachedDate"));
assertEquals(1, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, "Clinton"))));
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(sourceTableName);
updateInput.setRecords(List.of(new QRecord().withValue("id", 6).withValue("noOfShoes", 503)));
new UpdateAction().execute(updateInput);
updateInput = new UpdateInput();
updateInput.setTableName(cacheTableName);
updateInput.setRecords(List.of(getOutput.getRecord().withValue("cachedDate", Instant.parse("2001-01-01T00:00:00Z"))));
new UpdateAction().execute(updateInput);
getInput = new GetInput();
getInput.setTableName(cacheTableName);
getInput.setUniqueKey(Map.of("firstName", "Bill", "lastName", "Clinton"));
getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
assertEquals(503, getOutput.getRecord().getValue("noOfShoes"));
assertEquals(0, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, "Clinton"))));
}
///////////////////////////////////////////////////////////////////////////
// request a row that doesn't exist in cache or source, should miss both //
///////////////////////////////////////////////////////////////////////////
{
GetInput getInput = new GetInput();
getInput.setTableName(cacheTableName);
getInput.setUniqueKey(Map.of("firstName", "John", "lastName", "McCain"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNull(getOutput.getRecord());
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// update the record in the source table - then re-get from cache table - shouldn't see new value. //
/////////////////////////////////////////////////////////////////////////////////////////////////////
{
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(sourceTableName);
updateInput.setRecords(List.of(new QRecord().withValue("id", 1).withValue("noOfShoes", 6)));
new UpdateAction().execute(updateInput);
GetInput getInput = new GetInput();
getInput.setTableName(cacheTableName);
getInput.setUniqueKey(Map.of("firstName", "George", "lastName", "Washington"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
assertNotNull(getOutput.getRecord().getValue("cachedDate"));
assertEquals(5, getOutput.getRecord().getValue("noOfShoes"));
}
///////////////////////////////////////////////////////////////////////////
// delete the cached record; re-get, and we should see the updated value //
///////////////////////////////////////////////////////////////////////////
{
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(cacheTableName);
deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, "George")));
new DeleteAction().execute(deleteInput);
GetInput getInput = new GetInput();
getInput.setTableName(cacheTableName);
getInput.setUniqueKey(Map.of("firstName", "George", "lastName", "Washington"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
assertNotNull(getOutput.getRecord().getValue("cachedDate"));
assertEquals(6, getOutput.getRecord().getValue("noOfShoes"));
}
///////////////////////////////////////////////////////////////////
// update the source record; see that it isn't updated in cache. //
///////////////////////////////////////////////////////////////////
{
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(sourceTableName);
updateInput.setRecords(List.of(new QRecord().withValue("id", 1).withValue("noOfShoes", 7)));
new UpdateAction().execute(updateInput);
GetInput getInput = new GetInput();
getInput.setTableName(cacheTableName);
getInput.setUniqueKey(Map.of("firstName", "George", "lastName", "Washington"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
assertNotNull(getOutput.getRecord().getValue("cachedDate"));
assertEquals(6, getOutput.getRecord().getValue("noOfShoes"));
///////////////////////////////////////////////////////////////////////
// then artificially move back the cachedDate in the cache table. //
// then re-get from cache table, and we should see the updated value //
///////////////////////////////////////////////////////////////////////
updateInput = new UpdateInput();
updateInput.setTableName(cacheTableName);
updateInput.setRecords(List.of(getOutput.getRecord().withValue("cachedDate", Instant.parse("2001-01-01T00:00:00Z"))));
new UpdateAction().execute(updateInput);
getOutput = new GetAction().execute(getInput);
assertEquals(7, getOutput.getRecord().getValue("noOfShoes"));
}
/////////////////////////////////////////////////
// should only be 1 cache record at this point //
/////////////////////////////////////////////////
assertEquals(1, TestUtils.queryTable(QContext.getQInstance(), cacheTableName).size());
//////////////////////////////////////////////////////////////////////
// delete the source record - it will still be in the cache though. //
//////////////////////////////////////////////////////////////////////
{
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(sourceTableName);
deleteInput.setPrimaryKeys(List.of(1));
new DeleteAction().execute(deleteInput);
GetInput getInput = new GetInput();
getInput.setTableName(cacheTableName);
getInput.setUniqueKey(Map.of("firstName", "George", "lastName", "Washington"));
GetOutput getOutput = new GetAction().execute(getInput);
assertNotNull(getOutput.getRecord());
////////////////////////////////////////////////////////////////////
// then artificially move back the cachedDate in the cache table. //
// then re-get from cache table, and now it should go away //
////////////////////////////////////////////////////////////////////
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(cacheTableName);
updateInput.setRecords(List.of(getOutput.getRecord().withValue("cachedDate", Instant.parse("2001-01-01T00:00:00Z"))));
new UpdateAction().execute(updateInput);
getInput = new GetInput();
getInput.setTableName(cacheTableName);
getInput.setUniqueKey(Map.of("firstName", "George", "lastName", "Washington"));
getOutput = new GetAction().execute(getInput);
assertNull(getOutput.getRecord());
}
}
/*******************************************************************************
**
*******************************************************************************/
private static int countCachedRecordsDirectlyInBackend(String tableName, QQueryFilter filter) throws QException
{
List<QRecord> cachedRecords = MemoryRecordStore.getInstance().query(new QueryInput()
.withTableName(tableName)
.withFilter(filter));
return cachedRecords.size();
}
}

850
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryActionCacheHelperTest.java Normal file
View File

@ -0,0 +1,850 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.tables.helpers;
import java.time.Instant;
import java.util.List;
import java.util.stream.Stream;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
/*******************************************************************************
** Unit test for QueryActionCacheHelper
*******************************************************************************/
class QueryActionCacheHelperTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testUniqueKeyCacheSingleFieldUniqueKeySingleRecordUseCases() throws QException
{
QInstance qInstance = QContext.getQInstance();
String sourceTableName = TestUtils.TABLE_NAME_SHAPE;
String cacheTableName = TestUtils.TABLE_NAME_SHAPE_CACHE;
/////////////////////////////////////
// insert rows in the source table //
/////////////////////////////////////
TestUtils.insertRecords(qInstance, qInstance.getTable(sourceTableName), List.of(
new QRecord().withValue("id", 1).withValue("name", "Triangle").withValue("noOfSides", 3),
new QRecord().withValue("id", 2).withValue("name", "Square").withValue("noOfSides", 4),
new QRecord().withValue("id", 3).withValue("name", "Pentagon").withValue("noOfSides", 5),
new QRecord().withValue("id", 4).withValue("name", "ServerErrorGon").withValue("noOfSides", 503),
new QRecord().withValue("id", 5).withValue("name", "ManyGon").withValue("noOfSides", 999)
));
/////////////////////////////////////////////////////////////////////////////
// get from the table which caches it - confirm they are (magically) found //
/////////////////////////////////////////////////////////////////////////////
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Triangle")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertNotEquals(0, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(3, queryOutput.getRecords().get(0).getValue("noOfSides"));
}
////////////////////////////////////////////////////////////////////////////////////
// try to get from the table which caches it - it should be found, but not cached //
// because use case should filter out because of matching 503 //
////////////////////////////////////////////////////////////////////////////////////
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "ServerErrorGon")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "ManyGon")));
queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
}
///////////////////////////////////////////////////////////////////////////
// request a row that doesn't exist in cache or source, should miss both //
///////////////////////////////////////////////////////////////////////////
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Line"))); // lines aren't shapes :)
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(0, queryOutput.getRecords().size());
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// update the record in the source table - then re-get from cache table - shouldn't see new value. //
/////////////////////////////////////////////////////////////////////////////////////////////////////
{
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(sourceTableName);
updateInput.setRecords(List.of(new QRecord().withValue("id", 1).withValue("noOfSides", 6)));
new UpdateAction().execute(updateInput);
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Triangle")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertNotEquals(0, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(3, queryOutput.getRecords().get(0).getValue("noOfSides"));
}
///////////////////////////////////////////////////////////////////////////
// delete the cached record; re-get, and we should see the updated value //
///////////////////////////////////////////////////////////////////////////
{
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(cacheTableName);
deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Triangle")));
new DeleteAction().execute(deleteInput);
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Triangle")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertNotEquals(0, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(6, queryOutput.getRecords().get(0).getValue("noOfSides"));
}
///////////////////////////////////////////////////////////////////
// update the source record; see that it isn't updated in cache. //
///////////////////////////////////////////////////////////////////
{
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(sourceTableName);
updateInput.setRecords(List.of(new QRecord().withValue("id", 1).withValue("noOfSides", 7)));
new UpdateAction().execute(updateInput);
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Triangle")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertNotEquals(0, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(6, queryOutput.getRecords().get(0).getValue("noOfSides"));
///////////////////////////////////////////////////////////////////////
// then artificially move back the cachedDate in the cache table. //
// then re-get from cache table, and we should see the updated value //
///////////////////////////////////////////////////////////////////////
updateInput = new UpdateInput();
updateInput.setTableName(cacheTableName);
updateInput.setRecords(List.of(queryOutput.getRecords().get(0).withValue("cachedDate", Instant.parse("2001-01-01T00:00:00Z"))));
new UpdateAction().execute(updateInput);
queryOutput = new QueryAction().execute(queryInput);
assertEquals(7, queryOutput.getRecords().get(0).getValue("noOfSides"));
}
/////////////////////////////////////////////////
// should only be 1 cache record at this point //
/////////////////////////////////////////////////
assertEquals(1, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
//////////////////////////////////////////////////////////////////////
// delete the source record - it will still be in the cache though. //
//////////////////////////////////////////////////////////////////////
{
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(sourceTableName);
deleteInput.setPrimaryKeys(List.of(1));
new DeleteAction().execute(deleteInput);
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Triangle")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
assertEquals(1, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
////////////////////////////////////////////////////////////////////
// then artificially move back the cachedDate in the cache table. //
// then re-get from cache table, and now it should go away //
////////////////////////////////////////////////////////////////////
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(cacheTableName);
updateInput.setRecords(List.of(queryOutput.getRecords().get(0).withValue("cachedDate", Instant.parse("2001-01-01T00:00:00Z"))));
new UpdateAction().execute(updateInput);
queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Triangle")));
queryOutput = new QueryAction().execute(queryInput);
assertEquals(0, queryOutput.getRecords().size());
assertEquals(0, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testUniqueKeyCacheSingleFieldUniqueKeyMultiRecordUseCases() throws QException
{
QInstance qInstance = QContext.getQInstance();
String sourceTableName = TestUtils.TABLE_NAME_SHAPE;
String cacheTableName = TestUtils.TABLE_NAME_SHAPE_CACHE;
/////////////////////////////////////
// insert rows in the source table //
/////////////////////////////////////
TestUtils.insertRecords(qInstance.getTable(sourceTableName), List.of(
new QRecord().withValue("id", 1).withValue("name", "Triangle").withValue("noOfSides", 3),
new QRecord().withValue("id", 2).withValue("name", "Square").withValue("noOfSides", 4),
new QRecord().withValue("id", 3).withValue("name", "Pentagon").withValue("noOfSides", 5),
new QRecord().withValue("id", 4).withValue("name", "ServerErrorGon").withValue("noOfSides", 503),
new QRecord().withValue("id", 5).withValue("name", "ManyGon").withValue("noOfSides", 999)
));
/////////////////////////////////////////////////////////////////////////////
// get from the table which caches it - confirm they are (magically) found //
/////////////////////////////////////////////////////////////////////////////
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.IN, "Triangle", "Square", "Pentagon")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(3, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// try to get records through the cache table, which meet the conditions that cause them to not be cached. //
// so we should get results from the Query request - but - then let's go directly to the backend to confirm the records are not cached. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
{
assertEquals(3, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "ServerErrorGon")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.IN, "ManyGon", "ServerErrorGon")));
queryOutput = new QueryAction().execute(queryInput);
assertEquals(2, queryOutput.getRecords().size());
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.IN, "ManyGon", "Square")));
queryOutput = new QueryAction().execute(queryInput);
assertEquals(2, queryOutput.getRecords().size());
assertEquals(3, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
}
///////////////////////////////////////////////////////////////////////////
// request a row that doesn't exist in cache or source, should miss both //
///////////////////////////////////////////////////////////////////////////
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Line"))); // lines aren't shapes :)
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(0, queryOutput.getRecords().size());
}
//////////////////////////////////////////////////////////////////////////////////////////////////
// update one source record; delete another - query and should still find the previously cached //
//////////////////////////////////////////////////////////////////////////////////////////////////
{
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(sourceTableName);
updateInput.setRecords(List.of(new QRecord().withValue("id", 1).withValue("noOfSides", 6)));
new UpdateAction().execute(updateInput);
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(sourceTableName);
deleteInput.setPrimaryKeys(List.of(2)); // delete Square
new DeleteAction().execute(deleteInput);
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.IN, "Triangle", "Square", "Pentagon", "ServerErrorGon")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(4, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(3, queryOutput.getRecords().get(0).getValue("noOfSides"));
/////////////////////////////////////////////////////////////////////////////////////////////
// then artificially move back the cachedDate in the cache table. //
// then re-get from cache table, and we should see the updated value (and the deleted one) //
/////////////////////////////////////////////////////////////////////////////////////////////
updateInput = new UpdateInput();
updateInput.setTableName(cacheTableName);
updateInput.setRecords(Stream.of(1, 2, 3).map(id -> new QRecord().withValue("id", id).withValue("cachedDate", Instant.parse("2001-01-01T00:00:00Z"))).toList());
new UpdateAction().execute(updateInput);
queryOutput = new QueryAction().execute(queryInput);
assertEquals(3, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(6, queryOutput.getRecords().stream().filter(r -> r.getValueString("name").equals("Triangle")).findFirst().get().getValue("noOfSides"));
assertEquals(2, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testUniqueKeyCacheMultiFieldUniqueKeySingleRecordUseCases() throws QException
{
QInstance qInstance = QContext.getQInstance();
String sourceTableName = TestUtils.TABLE_NAME_PERSON_MEMORY;
String cacheTableName = TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE;
/////////////////////////////////////
// insert rows in the source table //
/////////////////////////////////////
TestUtils.insertRecords(qInstance.getTable(sourceTableName), List.of(
new QRecord().withValue("id", 1).withValue("firstName", "George").withValue("lastName", "Washington").withValue("noOfShoes", 5),
new QRecord().withValue("id", 2).withValue("firstName", "John").withValue("lastName", "Adams"),
new QRecord().withValue("id", 3).withValue("firstName", "Thomas").withValue("lastName", "Jefferson"),
new QRecord().withValue("id", 4).withValue("firstName", "James").withValue("lastName", "Garfield").withValue("noOfShoes", 503),
new QRecord().withValue("id", 5).withValue("firstName", "Abraham").withValue("lastName", "Lincoln").withValue("noOfShoes", 999),
new QRecord().withValue("id", 6).withValue("firstName", "Bill").withValue("lastName", "Clinton")
));
/////////////////////////////////////////////////////////////////////////////
// get from the table which caches it - confirm they are (magically) found //
/////////////////////////////////////////////////////////////////////////////
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(getFilterForPerson("George", "Washington"));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(5, queryOutput.getRecords().get(0).getValue("noOfShoes"));
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// try to get records through the cache table, which meet the conditions that cause them to not be cached. //
// so we should get results from the Get request - but - then let's go directly to the backend to confirm the records are not cached. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(getFilterForPerson("James", "Garfield"));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
queryInput.setFilter(getFilterForPerson("Abraham", "Lincoln"));
queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
assertEquals(0, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter(new QFilterCriteria("firstName", QCriteriaOperator.IN, "Abraham", "James"))));
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// fetch a record through the cache, so it gets cached. //
// then update the source record so that it meets the condition that doesn't allow it to be cached. //
// then expire the cached record. //
// then re-fetch through cache - which should see the expiration, re-fetch from source, and delete from cache. //
// assert record is no longer in cache. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(getFilterForPerson("Bill", "Clinton"));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(1, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, "Clinton"))));
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(sourceTableName);
updateInput.setRecords(List.of(new QRecord().withValue("id", 6).withValue("noOfShoes", 503)));
new UpdateAction().execute(updateInput);
updateInput = new UpdateInput();
updateInput.setTableName(cacheTableName);
updateInput.setRecords(List.of(queryOutput.getRecords().get(0).withValue("cachedDate", Instant.parse("2001-01-01T00:00:00Z"))));
new UpdateAction().execute(updateInput);
queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(503, queryOutput.getRecords().get(0).getValue("noOfShoes"));
assertEquals(0, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, "Clinton"))));
}
///////////////////////////////////////////////////////////////////////////
// request a row that doesn't exist in cache or source, should miss both //
///////////////////////////////////////////////////////////////////////////
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(getFilterForPerson("John", "McCain"));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(0, queryOutput.getRecords().size());
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// update the record in the source table - then re-get from cache table - shouldn't see new value. //
/////////////////////////////////////////////////////////////////////////////////////////////////////
{
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(sourceTableName);
updateInput.setRecords(List.of(new QRecord().withValue("id", 1).withValue("noOfShoes", 6)));
new UpdateAction().execute(updateInput);
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(getFilterForPerson("George", "Washington"));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(5, queryOutput.getRecords().get(0).getValue("noOfShoes"));
}
///////////////////////////////////////////////////////////////////////////
// delete the cached record; re-get, and we should see the updated value //
///////////////////////////////////////////////////////////////////////////
{
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(cacheTableName);
deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, "George")));
new DeleteAction().execute(deleteInput);
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(getFilterForPerson("George", "Washington"));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(6, queryOutput.getRecords().get(0).getValue("noOfShoes"));
}
///////////////////////////////////////////////////////////////////
// update the source record; see that it isn't updated in cache. //
///////////////////////////////////////////////////////////////////
{
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(sourceTableName);
updateInput.setRecords(List.of(new QRecord().withValue("id", 1).withValue("noOfShoes", 7)));
new UpdateAction().execute(updateInput);
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(getFilterForPerson("George", "Washington"));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(6, queryOutput.getRecords().get(0).getValue("noOfShoes"));
///////////////////////////////////////////////////////////////////////
// then artificially move back the cachedDate in the cache table. //
// then re-get from cache table, and we should see the updated value //
///////////////////////////////////////////////////////////////////////
updateInput = new UpdateInput();
updateInput.setTableName(cacheTableName);
updateInput.setRecords(List.of(queryOutput.getRecords().get(0).withValue("cachedDate", Instant.parse("2001-01-01T00:00:00Z"))));
new UpdateAction().execute(updateInput);
queryOutput = new QueryAction().execute(queryInput);
assertEquals(7, queryOutput.getRecords().get(0).getValue("noOfShoes"));
}
/////////////////////////////////////////////////
// should only be 1 cache record at this point //
/////////////////////////////////////////////////
assertEquals(1, TestUtils.queryTable(QContext.getQInstance(), cacheTableName).size());
//////////////////////////////////////////////////////////////////////
// delete the source record - it will still be in the cache though. //
//////////////////////////////////////////////////////////////////////
{
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(sourceTableName);
deleteInput.setPrimaryKeys(List.of(1));
new DeleteAction().execute(deleteInput);
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(getFilterForPerson("George", "Washington"));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
////////////////////////////////////////////////////////////////////
// then artificially move back the cachedDate in the cache table. //
// then re-get from cache table, and now it should go away //
////////////////////////////////////////////////////////////////////
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(cacheTableName);
updateInput.setRecords(List.of(queryOutput.getRecords().get(0).withValue("cachedDate", Instant.parse("2001-01-01T00:00:00Z"))));
new UpdateAction().execute(updateInput);
queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(getFilterForPerson("George", "Washington"));
queryOutput = new QueryAction().execute(queryInput);
assertEquals(0, queryOutput.getRecords().size());
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testUniqueKeyCacheMultiFieldUniqueKeyMultiRecordUseCases() throws QException
{
QInstance qInstance = QContext.getQInstance();
String sourceTableName = TestUtils.TABLE_NAME_PERSON_MEMORY;
String cacheTableName = TestUtils.TABLE_NAME_PERSON_MEMORY_CACHE;
/////////////////////////////////////
// insert rows in the source table //
/////////////////////////////////////
TestUtils.insertRecords(qInstance.getTable(sourceTableName), List.of(
new QRecord().withValue("id", 1).withValue("firstName", "George").withValue("lastName", "Washington").withValue("noOfShoes", 5),
new QRecord().withValue("id", 2).withValue("firstName", "John").withValue("lastName", "Adams"),
new QRecord().withValue("id", 3).withValue("firstName", "Thomas").withValue("lastName", "Jefferson"),
new QRecord().withValue("id", 4).withValue("firstName", "James").withValue("lastName", "Garfield").withValue("noOfShoes", 503),
new QRecord().withValue("id", 5).withValue("firstName", "Abraham").withValue("lastName", "Lincoln").withValue("noOfShoes", 999),
new QRecord().withValue("id", 6).withValue("firstName", "Bill").withValue("lastName", "Clinton")
));
/////////////////////////////////////////////////////////////////////////////
// get from the table which caches it - confirm they are (magically) found //
/////////////////////////////////////////////////////////////////////////////
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(getFilterForPersons(getFilterForPerson("George", "Washington"), getFilterForPerson("John", "Adams"), getFilterForPerson("Thomas", "Jefferson")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(3, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// try to get records through the cache table, which meet the conditions that cause them to not be cached. //
// so we should get results from the Query request - but - then let's go directly to the backend to confirm the records are not cached. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
{
assertEquals(3, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(getFilterForPersons(getFilterForPerson("James", "Garfield"), getFilterForPerson("Abraham", "Lincoln"), getFilterForPerson("Thomas", "Jefferson")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(3, queryOutput.getRecords().size());
assertEquals(3, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
}
///////////////////////////////////////////////////////////////////////////
// request a row that doesn't exist in cache or source, should miss both //
///////////////////////////////////////////////////////////////////////////
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(getFilterForPersons(getFilterForPerson("John", "McCain"), getFilterForPerson("John", "Kerry")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(0, queryOutput.getRecords().size());
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// fetch a record through the cache, so it gets cached. //
// then update the source record so that it meets the condition that doesn't allow it to be cached. //
// and delete another one. //
// then expire the cached records. //
// then re-fetch through cache - which should see the expiration, re-fetch from source, and delete from cache. //
// assert record is no longer in cache. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(getFilterForPersons(getFilterForPerson("George", "Washington"), getFilterForPerson("Bill", "Clinton")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(2, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(1, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, "Clinton"))));
assertEquals(1, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, "Washington"))));
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(sourceTableName);
updateInput.setRecords(List.of(new QRecord().withValue("id", 6).withValue("noOfShoes", 503)));
new UpdateAction().execute(updateInput);
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(sourceTableName);
deleteInput.setPrimaryKeys(List.of(1)); // delete Washington
new DeleteAction().execute(deleteInput);
updateInput = new UpdateInput();
updateInput.setTableName(cacheTableName);
updateInput.setRecords(Stream.of(1, 2, 3, 4).map(id -> new QRecord().withValue("id", id).withValue("cachedDate", Instant.parse("2001-01-01T00:00:00Z"))).toList());
new UpdateAction().execute(updateInput);
queryOutput = new QueryAction().execute(queryInput);
assertEquals(1, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(503, queryOutput.getRecords().stream().filter(r -> r.getValueString("lastName").equals("Clinton")).findFirst().get().getValue("noOfShoes"));
assertEquals(0, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, "Clinton"))));
assertEquals(0, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, "Washington"))));
}
/*
//////////////////////////////////////////////////////////////////////////////////////////////////
// update one source record; delete another - query and should still find the previously cached //
//////////////////////////////////////////////////////////////////////////////////////////////////
{
UpdateInput updateInput = new UpdateInput();
updateInput.setTableName(sourceTableName);
updateInput.setRecords(List.of(new QRecord().withValue("id", 1).withValue("noOfSides", 6)));
new UpdateAction().execute(updateInput);
DeleteInput deleteInput = new DeleteInput();
deleteInput.setTableName(sourceTableName);
deleteInput.setPrimaryKeys(List.of(2)); // delete Square
new DeleteAction().execute(deleteInput);
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.IN, "Triangle", "Square", "Pentagon", "ServerErrorGon")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(4, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(3, queryOutput.getRecords().get(0).getValue("noOfSides"));
/////////////////////////////////////////////////////////////////////////////////////////////
// then artificially move back the cachedDate in the cache table. //
// then re-get from cache table, and we should see the updated value (and the deleted one) //
/////////////////////////////////////////////////////////////////////////////////////////////
updateInput = new UpdateInput();
updateInput.setTableName(cacheTableName);
updateInput.setRecords(Stream.of(1, 2, 3).map(id -> new QRecord().withValue("id", id).withValue("cachedDate", Instant.parse("2001-01-01T00:00:00Z"))).toList());
new UpdateAction().execute(updateInput);
queryOutput = new QueryAction().execute(queryInput);
assertEquals(3, queryOutput.getRecords().size());
assertNotNull(queryOutput.getRecords().get(0).getValue("cachedDate"));
assertEquals(6, queryOutput.getRecords().stream().filter(r -> r.getValueString("name").equals("Triangle")).findFirst().get().getValue("noOfSides"));
assertEquals(2, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
}
*/
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testUniqueKeyCacheNonCachingUseCases() throws QException
{
QInstance qInstance = QContext.getQInstance();
String sourceTableName = TestUtils.TABLE_NAME_SHAPE;
String cacheTableName = TestUtils.TABLE_NAME_SHAPE_CACHE;
/////////////////////////////////////
// insert rows in the source table //
/////////////////////////////////////
TestUtils.insertRecords(qInstance.getTable(sourceTableName), List.of(
new QRecord().withValue("id", 1).withValue("name", "Triangle").withValue("noOfSides", 3),
new QRecord().withValue("id", 2).withValue("name", "Square").withValue("noOfSides", 4),
new QRecord().withValue("id", 3).withValue("name", "Pentagon").withValue("noOfSides", 5),
new QRecord().withValue("id", 4).withValue("name", "ServerErrorGon").withValue("noOfSides", 503),
new QRecord().withValue("id", 5).withValue("name", "ManyGon").withValue("noOfSides", 999)));
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// do queries on the cache table that we aren't allowed to do caching with - confirm that cache remains empty //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
{
///////////////
// no filter //
///////////////
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(0, queryOutput.getRecords().size());
assertEquals(0, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
}
{
//////////////////////////////
// unique key not in filter //
//////////////////////////////
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("noOfSides", QCriteriaOperator.LESS_THAN_OR_EQUALS, 5)));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(0, queryOutput.getRecords().size());
assertEquals(0, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
}
{
////////////////////////////////////////////////
// unsupported operator in filter on UK field //
////////////////////////////////////////////////
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.STARTS_WITH, "T")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(0, queryOutput.getRecords().size());
assertEquals(0, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
}
{
///////////////////////////////////////////////////////////////////////////
// an AND sub-filter //
// (technically we could do this, since only 1 sub-filter, but we don't) //
///////////////////////////////////////////////////////////////////////////
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter().withSubFilters(List.of(
new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Triangle"))
)));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(0, queryOutput.getRecords().size());
assertEquals(0, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
}
{
//////////////////////////////////////////////
// an OR sub-filter, but unsupported fields //
//////////////////////////////////////////////
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR).withSubFilters(List.of(
new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Triangle")),
new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 3))
)));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(0, queryOutput.getRecords().size());
assertEquals(0, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
}
{
////////////////////////////////////////////////////////////////////////////////////////////////////////
// an OR sub-filter, but with unsupported operator (IN - supported w/o subqueries, but not like this) //
// (technically we could do this, since only 1 sub-filter, but we don't) //
////////////////////////////////////////////////////////////////////////////////////////////////////////
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR).withSubFilters(List.of(
new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.IN, "Triangle", "Square"))
)));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(0, queryOutput.getRecords().size());
assertEquals(0, countCachedRecordsDirectlyInBackend(cacheTableName, new QQueryFilter()));
}
///////////////////////////////////////////////////////////////////////////////////////////
// finally - queries that DO hit cache (so note, cache will stop being empty after here) //
///////////////////////////////////////////////////////////////////////////////////////////
{
///////////////////////////////////////////////////////////
// an OR sub-filter, with supported ops, and UKey fields //
///////////////////////////////////////////////////////////
QueryInput queryInput = new QueryInput();
queryInput.setTableName(cacheTableName);
queryInput.setFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR).withSubFilters(List.of(
new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Triangle")),
new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Square"))
)));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertEquals(2, queryOutput.getRecords().size());
}
}
/*******************************************************************************
**
*******************************************************************************/
private static QQueryFilter getFilterForPerson(String firstName, String lastName)
{
return new QQueryFilter(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, firstName), new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, lastName));
}
/*******************************************************************************
**
*******************************************************************************/
private QQueryFilter getFilterForPersons(QQueryFilter... subFilters)
{
QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
for(QQueryFilter subFilter : subFilters)
{
filter.addSubFilter(subFilter);
}
return (filter);
}
/*******************************************************************************
**
*******************************************************************************/
private static int countCachedRecordsDirectlyInBackend(String tableName, QQueryFilter filter) throws QException
{
List<QRecord> cachedRecords = MemoryRecordStore.getInstance().query(new QueryInput()
.withTableName(tableName)
.withFilter(filter));
return cachedRecords.size();
}
}

44
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -129,6 +129,7 @@ public class TestUtils
public static final String TABLE_NAME_PERSON = "person"; public static final String TABLE_NAME_PERSON = "person";
public static final String TABLE_NAME_SHAPE = "shape"; public static final String TABLE_NAME_SHAPE = "shape";
public static final String TABLE_NAME_SHAPE_CACHE = "shapeCache";
public static final String TABLE_NAME_ORDER = "order"; public static final String TABLE_NAME_ORDER = "order";
public static final String TABLE_NAME_LINE_ITEM = "orderLine"; public static final String TABLE_NAME_LINE_ITEM = "orderLine";
public static final String TABLE_NAME_LINE_ITEM_EXTRINSIC = "orderLineExtrinsic"; public static final String TABLE_NAME_LINE_ITEM_EXTRINSIC = "orderLineExtrinsic";
@ -185,6 +186,7 @@ public class TestUtils
qInstance.addTable(definePersonMemoryCacheTable()); qInstance.addTable(definePersonMemoryCacheTable());
qInstance.addTable(defineTableIdAndNameOnly()); qInstance.addTable(defineTableIdAndNameOnly());
qInstance.addTable(defineTableShape()); qInstance.addTable(defineTableShape());
qInstance.addTable(defineShapeCacheTable());
qInstance.addTable(defineTableBasepull()); qInstance.addTable(defineTableBasepull());
qInstance.addTable(defineTableOrder()); qInstance.addTable(defineTableOrder());
qInstance.addTable(defineTableLineItem()); qInstance.addTable(defineTableLineItem());
@ -338,8 +340,7 @@ public class TestUtils
private static QAutomationProviderMetaData definePollingAutomationProvider() private static QAutomationProviderMetaData definePollingAutomationProvider()
{ {
return (new PollingAutomationProviderMetaData() return (new PollingAutomationProviderMetaData()
.withName(POLLING_AUTOMATION) .withName(POLLING_AUTOMATION));
);
} }
@ -847,8 +848,43 @@ public class TestUtils
.withCacheSourceMisses(false) .withCacheSourceMisses(false)
.withExcludeRecordsMatching(List.of( .withExcludeRecordsMatching(List.of(
new QQueryFilter( new QQueryFilter(
new QFilterCriteria("firstName", QCriteriaOperator.CONTAINS, "503"), new QFilterCriteria("noOfShoes", QCriteriaOperator.EQUALS, "503"),
new QFilterCriteria("firstName", QCriteriaOperator.CONTAINS, "999") new QFilterCriteria("noOfShoes", QCriteriaOperator.EQUALS, "999")
).withBooleanOperator(QQueryFilter.BooleanOperator.OR)
)
))
);
}
/*******************************************************************************
** Define another version of the 'shape' table, also in-memory, and as a
** cache on the other in-memory one...
*******************************************************************************/
public static QTableMetaData defineShapeCacheTable()
{
UniqueKey uniqueKey = new UniqueKey("name");
return (new QTableMetaData()
.withName(TABLE_NAME_SHAPE_CACHE)
.withBackendName(MEMORY_BACKEND_NAME)
.withPrimaryKeyField("id")
.withUniqueKey(uniqueKey)
.withFields(TestUtils.defineTableShape().getFields()))
.withField(new QFieldMetaData("cachedDate", QFieldType.DATE_TIME))
.withCacheOf(new CacheOf()
.withSourceTable(TABLE_NAME_SHAPE)
.withCachedDateFieldName("cachedDate")
.withExpirationSeconds(60)
.withUseCase(new CacheUseCase()
.withType(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY)
.withSourceUniqueKey(uniqueKey)
.withCacheUniqueKey(uniqueKey)
.withCacheSourceMisses(false)
.withExcludeRecordsMatching(List.of(
new QQueryFilter(
new QFilterCriteria("noOfSides", QCriteriaOperator.EQUALS, 503),
new QFilterCriteria("noOfSides", QCriteriaOperator.EQUALS, 999)
).withBooleanOperator(QQueryFilter.BooleanOperator.OR) ).withBooleanOperator(QQueryFilter.BooleanOperator.OR)
) )
)) ))