From 7b141abcec5dbbd6d85ea196b947e8389f99757a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 15 Jan 2024 20:21:16 -0600 Subject: [PATCH] CE-781 Add logQuery, queryStats, actionTimeouts to MongoDB; fix many query operators while adding test coverage --- .../actions/AbstractMongoDBAction.java | 193 +++- .../actions/MongoDBAggregateAction.java | 32 +- .../mongodb/actions/MongoDBCountAction.java | 34 +- .../mongodb/actions/MongoDBDeleteAction.java | 9 + .../mongodb/actions/MongoDBInsertAction.java | 20 +- .../mongodb/actions/MongoDBQueryAction.java | 37 +- .../mongodb/actions/MongoDBUpdateAction.java | 15 +- .../mongodb/actions/TimeoutCanceller.java | 68 ++ .../qqq/backend/module/mongodb/BaseTest.java | 16 +- .../qqq/backend/module/mongodb/TestUtils.java | 260 +++++- .../actions/MongoDBCountActionTest.java | 2 +- .../actions/MongoDBDeleteActionTest.java | 2 +- .../actions/MongoDBInsertActionTest.java | 2 +- .../actions/MongoDBQueryActionTest.java | 855 +++++++++++++++++- .../actions/MongoDBTransactionTest.java | 114 +++ .../actions/MongoDBUpdateActionTest.java | 44 +- 16 files changed, 1608 insertions(+), 95 deletions(-) create mode 100644 qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/TimeoutCanceller.java create mode 100644 qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBTransactionTest.java diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java index 76c99546..9364bf9a 100644 --- a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.Set; import java.util.regex.Pattern; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -40,14 +41,17 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.querystats.QueryStat; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -63,6 +67,7 @@ import com.mongodb.client.model.Filters; import org.bson.Document; import org.bson.conversions.Bson; import org.bson.types.ObjectId; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -72,6 +77,8 @@ public class AbstractMongoDBAction { private static final QLogger LOG = QLogger.getLogger(AbstractMongoDBAction.class); + protected QueryStat queryStat; + /******************************************************************************* @@ -137,6 +144,11 @@ public class AbstractMongoDBAction *******************************************************************************/ protected String getBackendTableName(QTableMetaData table) { + if(table == null) + { + return (null); + } + if(table.getBackendDetails() != null) { String backendTableName = ((MongoDBTableBackendDetails) table.getBackendDetails()).getTableName(); @@ -368,7 +380,15 @@ public class AbstractMongoDBAction } Bson searchQueryForSecurity = makeSearchQueryDocumentWithoutSecurity(table, securityFilter); - return (Filters.and(searchQueryWithoutSecurity, searchQueryForSecurity)); + + if(searchQueryWithoutSecurity.toBsonDocument().isEmpty()) + { + return (searchQueryForSecurity); + } + else + { + return (Filters.and(searchQueryWithoutSecurity, searchQueryForSecurity)); + } } @@ -524,6 +544,31 @@ public class AbstractMongoDBAction QFieldMetaData field = table.getField(criteria.getFieldName()); String fieldBackendName = getFieldBackendName(field); + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // replace any expression-type values with their evaluation // + // also, "scrub" non-expression values, which type-converts them (e.g., strings in various supported date formats become LocalDate) // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ListIterator valueListIterator = values.listIterator(); + while(valueListIterator.hasNext()) + { + Serializable value = valueListIterator.next(); + if(value instanceof AbstractFilterExpression expression) + { + valueListIterator.set(expression.evaluate()); + } + /* + todo - is this needed?? + else + { + Serializable scrubbedValue = scrubValue(field, value); + valueListIterator.set(scrubbedValue); + } + */ + } + + ///////////////////////////////////////////////////////////////////////////////////////// + // make sure any values we're going to run against the primary key (_id) are ObjectIds // + ///////////////////////////////////////////////////////////////////////////////////////// if(field.getName().equals(table.getPrimaryKeyField())) { ListIterator iterator = values.listIterator(); @@ -534,37 +579,53 @@ public class AbstractMongoDBAction } } - Serializable value0 = values.get(0); + //////// + // :( // + //////// + if(StringUtils.hasContent(criteria.getOtherFieldName())) + { + throw (new IllegalArgumentException("A mongodb query with an 'otherFieldName' specified is not currently supported.")); + } + criteriaFilters.add(switch(criteria.getOperator()) { - case EQUALS -> Filters.eq(fieldBackendName, value0); - case NOT_EQUALS -> Filters.ne(fieldBackendName, value0); + case EQUALS -> Filters.eq(fieldBackendName, getValue(values, 0)); + + case NOT_EQUALS -> Filters.and( + Filters.ne(fieldBackendName, getValue(values, 0)), + + //////////////////////////////////////////////////////////////////////////////////////////// + // to match RDBMS and other QQQ backends, consider a null to not match a not-equals query // + //////////////////////////////////////////////////////////////////////////////////////////// + Filters.not(Filters.eq(fieldBackendName, null)) + ); + case NOT_EQUALS_OR_IS_NULL -> Filters.or( Filters.eq(fieldBackendName, null), - Filters.ne(fieldBackendName, value0) + Filters.ne(fieldBackendName, getValue(values, 0)) ); case IN -> filterIn(fieldBackendName, values); - case NOT_IN -> Filters.not(filterIn(fieldBackendName, values)); + case NOT_IN -> Filters.nor(filterIn(fieldBackendName, values)); case IS_NULL_OR_IN -> Filters.or( Filters.eq(fieldBackendName, null), filterIn(fieldBackendName, values) ); - case LIKE -> filterRegex(fieldBackendName, null, ValueUtils.getValueAsString(value0).replaceAll("%", ".*"), null); - case NOT_LIKE -> Filters.not(filterRegex(fieldBackendName, null, ValueUtils.getValueAsString(value0).replaceAll("%", ".*"), null)); - case STARTS_WITH -> filterRegex(fieldBackendName, null, value0, ".*"); - case ENDS_WITH -> filterRegex(fieldBackendName, ".*", value0, null); - case CONTAINS -> filterRegex(fieldBackendName, ".*", value0, ".*"); - case NOT_STARTS_WITH -> Filters.not(filterRegex(fieldBackendName, null, value0, ".*")); - case NOT_ENDS_WITH -> Filters.not(filterRegex(fieldBackendName, ".*", value0, null)); - case NOT_CONTAINS -> Filters.not(filterRegex(fieldBackendName, ".*", value0, ".*")); - case LESS_THAN -> Filters.lt(fieldBackendName, value0); - case LESS_THAN_OR_EQUALS -> Filters.lte(fieldBackendName, value0); - case GREATER_THAN -> Filters.gt(fieldBackendName, value0); - case GREATER_THAN_OR_EQUALS -> Filters.gte(fieldBackendName, value0); + case LIKE -> filterRegex(fieldBackendName, null, ValueUtils.getValueAsString(getValue(values, 0)).replaceAll("%", ".*"), null); + case NOT_LIKE -> Filters.nor(filterRegex(fieldBackendName, null, ValueUtils.getValueAsString(getValue(values, 0)).replaceAll("%", ".*"), null)); + case STARTS_WITH -> filterRegex(fieldBackendName, null, getValue(values, 0), ".*"); + case ENDS_WITH -> filterRegex(fieldBackendName, ".*", getValue(values, 0), null); + case CONTAINS -> filterRegex(fieldBackendName, ".*", getValue(values, 0), ".*"); + case NOT_STARTS_WITH -> Filters.nor(filterRegex(fieldBackendName, null, getValue(values, 0), ".*")); + case NOT_ENDS_WITH -> Filters.nor(filterRegex(fieldBackendName, ".*", getValue(values, 0), null)); + case NOT_CONTAINS -> Filters.nor(filterRegex(fieldBackendName, ".*", getValue(values, 0), ".*")); + case LESS_THAN -> Filters.lt(fieldBackendName, getValue(values, 0)); + case LESS_THAN_OR_EQUALS -> Filters.lte(fieldBackendName, getValue(values, 0)); + case GREATER_THAN -> Filters.gt(fieldBackendName, getValue(values, 0)); + case GREATER_THAN_OR_EQUALS -> Filters.gte(fieldBackendName, getValue(values, 0)); case IS_BLANK -> filterIsBlank(fieldBackendName); - case IS_NOT_BLANK -> Filters.not(filterIsBlank(fieldBackendName)); + case IS_NOT_BLANK -> Filters.nor(filterIsBlank(fieldBackendName)); case BETWEEN -> filterBetween(fieldBackendName, values); - case NOT_BETWEEN -> Filters.not(filterBetween(fieldBackendName, values)); + case NOT_BETWEEN -> Filters.nor(filterBetween(fieldBackendName, values)); }); } @@ -585,6 +646,21 @@ public class AbstractMongoDBAction + /******************************************************************************* + ** + *******************************************************************************/ + private static Serializable getValue(List values, int i) + { + if(values == null || values.size() <= i) + { + throw new IllegalArgumentException("Incorrect number of values given for criteria"); + } + + return (values.get(i)); + } + + + /******************************************************************************* ** build a bson filter doing a regex (e.g., for LIKE, STARTS_WITH, etc) *******************************************************************************/ @@ -600,7 +676,7 @@ public class AbstractMongoDBAction suffix = ""; } - String fullRegex = prefix + Pattern.quote(ValueUtils.getValueAsString(mainRegex) + suffix); + String fullRegex = prefix + ValueUtils.getValueAsString(mainRegex + suffix); return (Filters.regex(fieldBackendName, Pattern.compile(fullRegex))); } @@ -622,8 +698,8 @@ public class AbstractMongoDBAction private static Bson filterBetween(String fieldBackendName, List values) { return Filters.and( - Filters.gte(fieldBackendName, values.get(0)), - Filters.lte(fieldBackendName, values.get(1)) + Filters.gte(fieldBackendName, getValue(values, 0)), + Filters.lte(fieldBackendName, getValue(values, 1)) ); } @@ -639,4 +715,75 @@ public class AbstractMongoDBAction Filters.eq(fieldBackendName, "") ); } + + + + /******************************************************************************* + ** Getter for queryStat + *******************************************************************************/ + public QueryStat getQueryStat() + { + return (this.queryStat); + } + + + + /******************************************************************************* + ** Setter for queryStat + *******************************************************************************/ + public void setQueryStat(QueryStat queryStat) + { + this.queryStat = queryStat; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void setQueryInQueryStat(Bson query) + { + if(queryStat != null && query != null) + { + queryStat.setQueryText(query.toString()); + + //////////////////////////////////////////////////////////////// + // todo - if we support joins in the future, do them here too // + //////////////////////////////////////////////////////////////// + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void logQuery(String tableName, String actionName, List query, Long queryStartTime) + { + + if(System.getProperty("qqq.mongodb.logQueries", "false").equals("true")) + { + try + { + if(System.getProperty("qqq.mongodb.logQueries.output", "logger").equalsIgnoreCase("system.out")) + { + System.out.println("Table: " + tableName + ", Action: " + actionName + ", Query: " + query); + + if(queryStartTime != null) + { + System.out.println("Query Took [" + QValueFormatter.formatValue(DisplayFormat.COMMAS, (System.currentTimeMillis() - queryStartTime)) + "] ms"); + } + } + else + { + LOG.debug("Running Query", logPair("table", tableName), logPair("action", actionName), logPair("query", query), logPair("millis", queryStartTime == null ? null : (System.currentTimeMillis() - queryStartTime))); + } + } + catch(Exception e) + { + LOG.debug("Error logging query...", e); + } + } + } + } diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateAction.java index 60b34fde..04da2115 100644 --- a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateAction.java +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateAction.java @@ -24,8 +24,11 @@ package com.kingsrook.qqq.backend.module.mongodb.actions; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface; +import com.kingsrook.qqq.backend.core.actions.tables.helpers.ActionTimeoutHelper; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; @@ -61,7 +64,7 @@ public class MongoDBAggregateAction extends AbstractMongoDBAction implements Agg { private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class); - // todo? private ActionTimeoutHelper actionTimeoutHelper; + private ActionTimeoutHelper actionTimeoutHelper; @@ -73,6 +76,9 @@ public class MongoDBAggregateAction extends AbstractMongoDBAction implements Agg { MongoClientContainer mongoClientContainer = null; + Long queryStartTime = System.currentTimeMillis(); + List queryToLog = new ArrayList<>(); + try { AggregateOutput aggregateOutput = new AggregateOutput(); @@ -87,6 +93,12 @@ public class MongoDBAggregateAction extends AbstractMongoDBAction implements Agg QQueryFilter filter = aggregateInput.getFilter(); Bson searchQuery = makeSearchQueryDocument(table, filter); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up & start an actionTimeoutHelper (note, internally it'll deal with the time being null or negative as meaning not to timeout) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + actionTimeoutHelper = new ActionTimeoutHelper(aggregateInput.getTimeoutSeconds(), TimeUnit.SECONDS, new TimeoutCanceller(mongoClientContainer)); + actionTimeoutHelper.start(); + ///////////////////////////////////////////////////////////////////////// // we have to submit a list of BSON objects to the aggregate function. // // the first one is the search query // @@ -94,6 +106,8 @@ public class MongoDBAggregateAction extends AbstractMongoDBAction implements Agg ///////////////////////////////////////////////////////////////////////// List bsonList = new ArrayList<>(); bsonList.add(Aggregates.match(searchQuery)); + setQueryInQueryStat(searchQuery); + queryToLog = bsonList; ////////////////////////////////////////////////////////////////////////////////////// // if there are group-by fields, then we need to build a document with those fields // @@ -184,6 +198,12 @@ public class MongoDBAggregateAction extends AbstractMongoDBAction implements Agg ///////////////////// for(Document document : aggregates) { + ///////////////////////////////////////////////////////////////////////// + // once we've started getting results, go ahead and cancel the timeout // + ///////////////////////////////////////////////////////////////////////// + actionTimeoutHelper.cancel(); + setQueryStatFirstResultTime(); + AggregateResult result = new AggregateResult(); results.add(result); @@ -222,13 +242,16 @@ public class MongoDBAggregateAction extends AbstractMongoDBAction implements Agg } catch(Exception e) { - /* if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout()) { - setCountStatFirstResultTime(); + setQueryStatFirstResultTime(); throw (new QUserFacingException("Aggregate timed out.")); } + /* + ///////////////////////////////////////////////////////////////////////////////////// + // this was copied from RDBMS - not sure where/how/if it's being used there though // + ///////////////////////////////////////////////////////////////////////////////////// if(isCancelled) { throw (new QUserFacingException("Aggregate was cancelled.")); @@ -239,8 +262,9 @@ public class MongoDBAggregateAction extends AbstractMongoDBAction implements Agg throw new QException("Error executing aggregate", e); } finally - { + logQuery(getBackendTableName(aggregateInput.getTable()), "aggregate", queryToLog, queryStartTime); + if(mongoClientContainer != null) { mongoClientContainer.closeIfNeeded(); diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountAction.java index 277977a7..93e8baee 100644 --- a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountAction.java +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountAction.java @@ -22,9 +22,13 @@ package com.kingsrook.qqq.backend.module.mongodb.actions; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.actions.tables.helpers.ActionTimeoutHelper; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; @@ -48,7 +52,7 @@ public class MongoDBCountAction extends AbstractMongoDBAction implements CountIn { private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class); - // todo? private ActionTimeoutHelper actionTimeoutHelper; + private ActionTimeoutHelper actionTimeoutHelper; @@ -59,6 +63,9 @@ public class MongoDBCountAction extends AbstractMongoDBAction implements CountIn { MongoClientContainer mongoClientContainer = null; + Long queryStartTime = System.currentTimeMillis(); + List queryToLog = new ArrayList<>(); + try { CountOutput countOutput = new CountOutput(); @@ -70,34 +77,43 @@ public class MongoDBCountAction extends AbstractMongoDBAction implements CountIn MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName()); MongoCollection collection = database.getCollection(backendTableName); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up & start an actionTimeoutHelper (note, internally it'll deal with the time being null or negative as meaning not to timeout) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + actionTimeoutHelper = new ActionTimeoutHelper(countInput.getTimeoutSeconds(), TimeUnit.SECONDS, new TimeoutCanceller(mongoClientContainer)); + actionTimeoutHelper.start(); + QQueryFilter filter = countInput.getFilter(); Bson searchQuery = makeSearchQueryDocument(table, filter); + queryToLog.add(searchQuery); + setQueryInQueryStat(searchQuery); List bsonList = List.of( Aggregates.match(searchQuery), Aggregates.group("_id", Accumulators.sum("count", 1))); - //////////////////////////////////////////////////////// - // todo - system property to control (like print-sql) // - //////////////////////////////////////////////////////// - // LOG.debug(bsonList.toString()); - AggregateIterable aggregate = collection.aggregate(mongoClientContainer.getMongoSession(), bsonList); Document document = aggregate.first(); countOutput.setCount(document == null ? 0 : document.get("count", Integer.class)); + actionTimeoutHelper.cancel(); + setQueryStatFirstResultTime(); + return (countOutput); } catch(Exception e) { - /* if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout()) { - setCountStatFirstResultTime(); + setQueryStatFirstResultTime(); throw (new QUserFacingException("Count timed out.")); } + /* + ///////////////////////////////////////////////////////////////////////////////////// + // this was copied from RDBMS - not sure where/how/if it's being used there though // + ///////////////////////////////////////////////////////////////////////////////////// if(isCancelled) { throw (new QUserFacingException("Count was cancelled.")); @@ -109,6 +125,8 @@ public class MongoDBCountAction extends AbstractMongoDBAction implements CountIn } finally { + logQuery(getBackendTableName(countInput.getTable()), "count", queryToLog, queryStartTime); + if(mongoClientContainer != null) { mongoClientContainer.closeIfNeeded(); diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteAction.java index 54806601..6284df2e 100644 --- a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteAction.java +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteAction.java @@ -22,6 +22,8 @@ package com.kingsrook.qqq.backend.module.mongodb.actions; +import java.util.ArrayList; +import java.util.List; import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -70,6 +72,9 @@ public class MongoDBDeleteAction extends AbstractMongoDBAction implements Delete { MongoClientContainer mongoClientContainer = null; + Long queryStartTime = System.currentTimeMillis(); + List queryToLog = new ArrayList<>(); + try { DeleteOutput deleteOutput = new DeleteOutput(); @@ -98,6 +103,8 @@ public class MongoDBDeleteAction extends AbstractMongoDBAction implements Delete return (deleteOutput); } + queryToLog.add(searchQuery); + //////////////////////////////////////////////////////// // todo - system property to control (like print-sql) // //////////////////////////////////////////////////////// @@ -119,6 +126,8 @@ public class MongoDBDeleteAction extends AbstractMongoDBAction implements Delete } finally { + logQuery(getBackendTableName(deleteInput.getTable()), "delete", queryToLog, queryStartTime); + if(mongoClientContainer != null) { mongoClientContainer.closeIfNeeded(); diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertAction.java index e385f570..d02a67d8 100644 --- a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertAction.java +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertAction.java @@ -38,6 +38,7 @@ import com.mongodb.client.MongoDatabase; import com.mongodb.client.result.InsertManyResult; import org.bson.BsonValue; import org.bson.Document; +import org.bson.conversions.Bson; /******************************************************************************* @@ -59,6 +60,9 @@ public class MongoDBInsertAction extends AbstractMongoDBAction implements Insert List outputRecords = new ArrayList<>(); rs.setRecords(outputRecords); + Long queryStartTime = System.currentTimeMillis(); + List queryToLog = new ArrayList<>(); + try { QTableMetaData table = insertInput.getTable(); @@ -69,10 +73,6 @@ public class MongoDBInsertAction extends AbstractMongoDBAction implements Insert MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName()); MongoCollection collection = database.getCollection(backendTableName); - ////////////////////////// - // todo - transaction?! // - ////////////////////////// - /////////////////////////////////////////////////////////////////////////// // page over input record list (assuming some size of batch is too big?) // /////////////////////////////////////////////////////////////////////////// @@ -88,7 +88,10 @@ public class MongoDBInsertAction extends AbstractMongoDBAction implements Insert { continue; } - documentList.add(recordToDocument(table, record)); + + Document document = recordToDocument(table, record); + documentList.add(document); + queryToLog.add(document); } ///////////////////////////////////// @@ -99,11 +102,6 @@ public class MongoDBInsertAction extends AbstractMongoDBAction implements Insert continue; } - //////////////////////////////////////////////////////// - // todo - system property to control (like print-sql) // - //////////////////////////////////////////////////////// - // LOG.debug(documentList); - /////////////////////////////////////////////// // actually do the insert // // todo - how are errors returned by mongo?? // @@ -134,6 +132,8 @@ public class MongoDBInsertAction extends AbstractMongoDBAction implements Insert } finally { + logQuery(getBackendTableName(insertInput.getTable()), "insert", queryToLog, queryStartTime); + if(mongoClientContainer != null) { mongoClientContainer.closeIfNeeded(); diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryAction.java index 25b433df..9b9f27e0 100644 --- a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryAction.java +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryAction.java @@ -22,8 +22,13 @@ package com.kingsrook.qqq.backend.module.mongodb.actions; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.actions.tables.helpers.ActionTimeoutHelper; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -48,7 +53,7 @@ public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryIn { private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class); - // todo? private ActionTimeoutHelper actionTimeoutHelper; + private ActionTimeoutHelper actionTimeoutHelper; @@ -59,6 +64,9 @@ public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryIn { MongoClientContainer mongoClientContainer = null; + Long queryStartTime = System.currentTimeMillis(); + List queryToLog = new ArrayList<>(); + try { QueryOutput queryOutput = new QueryOutput(queryInput); @@ -70,16 +78,19 @@ public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryIn MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName()); MongoCollection collection = database.getCollection(backendTableName); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up & start an actionTimeoutHelper (note, internally it'll deal with the time being null or negative as meaning not to timeout) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + actionTimeoutHelper = new ActionTimeoutHelper(queryInput.getTimeoutSeconds(), TimeUnit.SECONDS, new TimeoutCanceller(mongoClientContainer)); + actionTimeoutHelper.start(); + ///////////////////////// // set up filter/query // ///////////////////////// QQueryFilter filter = queryInput.getFilter(); Bson searchQuery = makeSearchQueryDocument(table, filter); - - //////////////////////////////////////////////////////// - // todo - system property to control (like print-sql) // - //////////////////////////////////////////////////////// - // LOG.debug(searchQuery); + queryToLog.add(searchQuery); + setQueryInQueryStat(searchQuery); //////////////////////////////////////////////////////////// // create cursor - further adjustments to it still follow // @@ -92,6 +103,7 @@ public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryIn if(filter != null && CollectionUtils.nullSafeHasContents(filter.getOrderBys())) { Document sortDocument = new Document(); + queryToLog.add(sortDocument); for(QFilterOrderBy orderBy : filter.getOrderBys()) { String fieldBackendName = getFieldBackendName(table.getField(orderBy.getFieldName())); @@ -121,6 +133,12 @@ public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryIn //////////////////////////////////////////// for(Document document : cursor) { + ///////////////////////////////////////////////////////////////////////// + // once we've started getting results, go ahead and cancel the timeout // + ///////////////////////////////////////////////////////////////////////// + actionTimeoutHelper.cancel(); + setQueryStatFirstResultTime(); + QRecord record = documentToRecord(table, document); queryOutput.addRecord(record); @@ -135,13 +153,16 @@ public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryIn } catch(Exception e) { - /* if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout()) { setQueryStatFirstResultTime(); throw (new QUserFacingException("Query timed out.")); } + /* + ///////////////////////////////////////////////////////////////////////////////////// + // this was copied from RDBMS - not sure where/how/if it's being used there though // + ///////////////////////////////////////////////////////////////////////////////////// if(isCancelled) { throw (new QUserFacingException("Query was cancelled.")); @@ -153,6 +174,8 @@ public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryIn } finally { + logQuery(getBackendTableName(queryInput.getTable()), "query", queryToLog, queryStartTime); + if(mongoClientContainer != null) { mongoClientContainer.closeIfNeeded(); diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateAction.java index 3642b6f3..92ec2571 100644 --- a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateAction.java +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateAction.java @@ -141,26 +141,29 @@ public class MongoDBUpdateAction extends AbstractMongoDBAction implements Update *******************************************************************************/ private void updateRecordsWithMatchingValuesAndFields(MongoClientContainer mongoClientContainer, MongoCollection collection, QTableMetaData table, List recordList, List fieldsBeingUpdated) { + Long queryStartTime = System.currentTimeMillis(); + List queryToLog = new ArrayList<>(); + QRecord firstRecord = recordList.get(0); List ids = recordList.stream().map(r -> new ObjectId(r.getValueString("id"))).toList(); Bson filter = Filters.in("_id", ids); + queryToLog.add(filter); List updates = new ArrayList<>(); for(String fieldName : fieldsBeingUpdated) { QFieldMetaData field = table.getField(fieldName); String fieldBackendName = getFieldBackendName(field); - updates.add(Updates.set(fieldBackendName, firstRecord.getValue(fieldName))); + Bson set = Updates.set(fieldBackendName, firstRecord.getValue(fieldName)); + updates.add(set); + queryToLog.add(set); } Bson changes = Updates.combine(updates); - //////////////////////////////////////////////////////// - // todo - system property to control (like print-sql) // - //////////////////////////////////////////////////////// - // LOG.debug(filter, changes); - UpdateResult updateResult = collection.updateMany(mongoClientContainer.getMongoSession(), filter, changes); // todo - anything with the output?? + + logQuery(getBackendTableName(table), "update", queryToLog, queryStartTime); } } diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/TimeoutCanceller.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/TimeoutCanceller.java new file mode 100644 index 00000000..39b8db11 --- /dev/null +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/TimeoutCanceller.java @@ -0,0 +1,68 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.logging.QLogger; + + +/******************************************************************************* + ** Helper to cancel statements that timeout. + *******************************************************************************/ +public class TimeoutCanceller implements Runnable +{ + private static final QLogger LOG = QLogger.getLogger(TimeoutCanceller.class); + private final MongoClientContainer mongoClientContainer; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TimeoutCanceller(MongoClientContainer mongoClientContainer) + { + this.mongoClientContainer = mongoClientContainer; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run() + { + try + { + mongoClientContainer.closeIfNeeded(); + LOG.info("Cancelled timed out query"); + } + catch(Exception e) + { + LOG.warn("Error trying to cancel statement after timeout", e); + } + + throw (new QRuntimeException("Statement timed out and was cancelled.")); + } +} diff --git a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/BaseTest.java b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/BaseTest.java index 2cea4bf6..fe937f3a 100644 --- a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/BaseTest.java +++ b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/BaseTest.java @@ -58,6 +58,8 @@ public class BaseTest @BeforeAll static void beforeAll() { + System.setProperty("qqq.mongodb.logQueries", "true"); + mongoDBContainer = new GenericContainer<>(DockerImageName.parse(MONGO_IMAGE)) .withEnv("MONGO_INITDB_ROOT_USERNAME", TestUtils.MONGO_USERNAME) .withEnv("MONGO_INITDB_ROOT_PASSWORD", TestUtils.MONGO_PASSWORD) @@ -92,6 +94,18 @@ public class BaseTest *******************************************************************************/ @AfterEach void baseAfterEach() + { + clearDatabase(); + + QContext.clear(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected static void clearDatabase() { /////////////////////////////////////// // clear test database between tests // @@ -99,8 +113,6 @@ public class BaseTest MongoClient mongoClient = getMongoClient(); MongoDatabase database = mongoClient.getDatabase(TestUtils.MONGO_DATABASE); database.drop(); - - QContext.clear(); } diff --git a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/TestUtils.java b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/TestUtils.java index 82e61ef1..9687dff1 100644 --- a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/TestUtils.java +++ b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/TestUtils.java @@ -22,11 +22,22 @@ package com.kingsrook.qqq.backend.module.mongodb; +import java.util.List; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBTableBackendDetails; @@ -34,6 +45,8 @@ import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBTableBacke /******************************************************************************* ** Test Utils class for this module + ** + ** Note - tons of copying from RDMBS... wouldn't it be nice to share?? *******************************************************************************/ public class TestUtils { @@ -41,6 +54,15 @@ public class TestUtils public static final String TABLE_NAME_PERSON = "personTable"; + public static final String TABLE_NAME_STORE = "store"; + public static final String TABLE_NAME_ORDER = "order"; + public static final String TABLE_NAME_ORDER_INSTRUCTIONS = "orderInstructions"; + public static final String TABLE_NAME_ITEM = "item"; + public static final String TABLE_NAME_ORDER_LINE = "orderLine"; + public static final String TABLE_NAME_LINE_ITEM_EXTRINSIC = "orderLineExtrinsic"; + public static final String TABLE_NAME_WAREHOUSE = "warehouse"; + public static final String TABLE_NAME_WAREHOUSE_STORE_INT = "warehouseStoreInt"; + public static final String SECURITY_KEY_STORE_ALL_ACCESS = "storeAllAccess"; public static final String MONGO_USERNAME = "mongoUser"; @@ -48,33 +70,6 @@ public class TestUtils public static final Integer MONGO_PORT = 27017; public static final String MONGO_DATABASE = "testDatabase"; - public static final String TEST_COLLECTION = "testTable"; - - - - /******************************************************************************* - ** - *******************************************************************************/ - @SuppressWarnings("unchecked") - public static void primeTestDatabase(String sqlFileName) throws Exception - { - /* - ConnectionManager connectionManager = new ConnectionManager(); - try(Connection connection = connectionManager.getConnection(TestUtils.defineBackend())) - { - InputStream primeTestDatabaseSqlStream = RDBMSActionTest.class.getResourceAsStream("/" + sqlFileName); - assertNotNull(primeTestDatabaseSqlStream); - List lines = (List) IOUtils.readLines(primeTestDatabaseSqlStream); - lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList(); - String joinedSQL = String.join("\n", lines); - for(String sql : joinedSQL.split(";")) - { - QueryManager.executeUpdate(connection, sql); - } - } - */ - } - /******************************************************************************* @@ -85,6 +80,8 @@ public class TestUtils QInstance qInstance = new QInstance(); qInstance.addBackend(defineBackend()); qInstance.addTable(defineTablePerson()); + qInstance.addPossibleValueSource(definePvsPerson()); + addOmsTablesAndJoins(qInstance); qInstance.setAuthentication(defineAuthentication()); return (qInstance); } @@ -116,7 +113,8 @@ public class TestUtils .withUsername(TestUtils.MONGO_USERNAME) .withPassword(TestUtils.MONGO_PASSWORD) .withAuthSourceDatabase("admin") - .withDatabaseName(TestUtils.MONGO_DATABASE)); + .withDatabaseName(TestUtils.MONGO_DATABASE) + .withTransactionsSupported(false)); } @@ -136,6 +134,7 @@ public class TestUtils .withField(new QFieldMetaData("id", QFieldType.STRING).withBackendName("_id")) .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("metaData.createDate")) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("metaData.modifyDate")) + .withField(new QFieldMetaData("seqNo", QFieldType.INTEGER)) .withField(new QFieldMetaData("firstName", QFieldType.STRING)) .withField(new QFieldMetaData("lastName", QFieldType.STRING)) .withField(new QFieldMetaData("birthDate", QFieldType.DATE)) @@ -145,7 +144,210 @@ public class TestUtils .withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER)) .withField(new QFieldMetaData("homeTown", QFieldType.STRING)) .withBackendDetails(new MongoDBTableBackendDetails() - .withTableName(TEST_COLLECTION)); + .withTableName(TABLE_NAME_PERSON)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QPossibleValueSource definePvsPerson() + { + return (new QPossibleValueSource() + .withName(TABLE_NAME_PERSON) + .withType(QPossibleValueSourceType.TABLE) + .withTableName(TABLE_NAME_PERSON) + .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY) + ); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void addOmsTablesAndJoins(QInstance qInstance) + { + qInstance.addTable(defineBaseTable(TABLE_NAME_STORE, "store") + .withRecordLabelFormat("%s") + .withRecordLabelFields("name") + .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("key")) + .withField(new QFieldMetaData("name", QFieldType.STRING)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER, "order") + .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeKey")) + .withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_ORDER_LINE).withJoinName("orderJoinOrderLine")) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ITEM).withJoinPath(List.of("orderJoinOrderLine", "orderLineJoinItem"))) + .withField(new QFieldMetaData("storeKey", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_STORE)) + .withField(new QFieldMetaData("billToPersonId", QFieldType.STRING).withPossibleValueSourceName(TABLE_NAME_PERSON)) + .withField(new QFieldMetaData("shipToPersonId", QFieldType.STRING).withPossibleValueSourceName(TABLE_NAME_PERSON)) + .withField(new QFieldMetaData("currentOrderInstructionsId", QFieldType.STRING).withPossibleValueSourceName(TABLE_NAME_PERSON)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER_INSTRUCTIONS, "order_instructions") + .withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(TABLE_NAME_STORE) + .withFieldName("order.storeKey") + .withJoinNameChain(List.of("orderInstructionsJoinOrder"))) + .withField(new QFieldMetaData("orderId", QFieldType.STRING)) + .withField(new QFieldMetaData("instructions", QFieldType.STRING)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_ITEM, "item") + .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeKey")) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER).withJoinPath(List.of("orderLineJoinItem", "orderJoinOrderLine"))) + .withField(new QFieldMetaData("sku", QFieldType.STRING)) + .withField(new QFieldMetaData("description", QFieldType.STRING)) + .withField(new QFieldMetaData("storeKey", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_STORE)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER_LINE, "order_line") + .withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(TABLE_NAME_STORE) + .withFieldName("order.storeKey") + .withJoinNameChain(List.of("orderJoinOrderLine"))) + .withAssociation(new Association().withName("extrinsics").withAssociatedTableName(TABLE_NAME_LINE_ITEM_EXTRINSIC).withJoinName("orderLineJoinLineItemExtrinsic")) + .withField(new QFieldMetaData("orderId", QFieldType.STRING)) + .withField(new QFieldMetaData("sku", QFieldType.STRING)) + .withField(new QFieldMetaData("storeKey", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_STORE)) + .withField(new QFieldMetaData("quantity", QFieldType.INTEGER)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_LINE_ITEM_EXTRINSIC, "line_item_extrinsic") + .withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(TABLE_NAME_STORE) + .withFieldName("order.storeKey") + .withJoinNameChain(List.of("orderJoinOrderLine", "orderLineJoinLineItemExtrinsic"))) + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("orderLineId", QFieldType.STRING)) + .withField(new QFieldMetaData("key", QFieldType.STRING)) + .withField(new QFieldMetaData("value", QFieldType.STRING)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_WAREHOUSE_STORE_INT, "warehouse_store_int") + .withField(new QFieldMetaData("warehouseId", QFieldType.STRING)) + .withField(new QFieldMetaData("storeKey", QFieldType.INTEGER)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_WAREHOUSE, "warehouse") + .withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(TABLE_NAME_STORE) + .withFieldName(TABLE_NAME_WAREHOUSE_STORE_INT + ".storeKey") + .withJoinNameChain(List.of(QJoinMetaData.makeInferredJoinName(TestUtils.TABLE_NAME_WAREHOUSE, TestUtils.TABLE_NAME_WAREHOUSE_STORE_INT))) + ) + .withField(new QFieldMetaData("name", QFieldType.STRING)) + ); + + qInstance.addJoin(new QJoinMetaData() + .withType(JoinType.ONE_TO_MANY) + .withLeftTable(TestUtils.TABLE_NAME_WAREHOUSE) + .withRightTable(TestUtils.TABLE_NAME_WAREHOUSE_STORE_INT) + .withInferredName() + .withJoinOn(new JoinOn("id", "warehouseId")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinStore") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_STORE) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("storeKey", "key")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinBillToPerson") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_PERSON) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("billToPersonId", "id")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinShipToPerson") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_PERSON) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("shipToPersonId", "id")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("itemJoinStore") + .withLeftTable(TABLE_NAME_ITEM) + .withRightTable(TABLE_NAME_STORE) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("storeKey", "key")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinOrderLine") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_ORDER_LINE) + .withType(JoinType.ONE_TO_MANY) + .withJoinOn(new JoinOn("id", "orderId")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderLineJoinItem") + .withLeftTable(TABLE_NAME_ORDER_LINE) + .withRightTable(TABLE_NAME_ITEM) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("sku", "sku")) + .withJoinOn(new JoinOn("storeKey", "storeKey")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderLineJoinLineItemExtrinsic") + .withLeftTable(TABLE_NAME_ORDER_LINE) + .withRightTable(TABLE_NAME_LINE_ITEM_EXTRINSIC) + .withType(JoinType.ONE_TO_MANY) + .withJoinOn(new JoinOn("id", "orderLineId")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinCurrentOrderInstructions") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_ORDER_INSTRUCTIONS) + .withType(JoinType.ONE_TO_ONE) + .withJoinOn(new JoinOn("currentOrderInstructionsId", "id")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderInstructionsJoinOrder") + .withLeftTable(TABLE_NAME_ORDER_INSTRUCTIONS) + .withRightTable(TABLE_NAME_ORDER) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("orderId", "id")) + ); + + qInstance.addPossibleValueSource(new QPossibleValueSource() + .withName("store") + .withType(QPossibleValueSourceType.TABLE) + .withTableName(TABLE_NAME_STORE) + .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY) + ); + + qInstance.addSecurityKeyType(new QSecurityKeyType() + .withName(TABLE_NAME_STORE) + .withAllAccessKeyName(SECURITY_KEY_STORE_ALL_ACCESS) + .withPossibleValueSourceName(TABLE_NAME_STORE)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QTableMetaData defineBaseTable(String tableName, String backendTableName) + { + return new QTableMetaData() + .withName(tableName) + .withBackendName(DEFAULT_BACKEND_NAME) + .withBackendDetails(new MongoDBTableBackendDetails().withTableName(backendTableName)) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.STRING)) + .withField(new QFieldMetaData("key", QFieldType.INTEGER)); } } diff --git a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountActionTest.java b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountActionTest.java index bc8ca654..8f520383 100644 --- a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountActionTest.java +++ b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountActionTest.java @@ -54,7 +54,7 @@ class MongoDBCountActionTest extends BaseTest // directly insert some mongo records // //////////////////////////////////////// MongoDatabase database = getMongoClient().getDatabase(TestUtils.MONGO_DATABASE); - MongoCollection collection = database.getCollection(TestUtils.TEST_COLLECTION); + MongoCollection collection = database.getCollection(TestUtils.TABLE_NAME_PERSON); collection.insertMany(List.of( Document.parse(""" {"firstName": "Darin", "lastName": "Kelkhoff"}"""), diff --git a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteActionTest.java b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteActionTest.java index 8c751a1b..2cd4a9c7 100644 --- a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteActionTest.java +++ b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteActionTest.java @@ -57,7 +57,7 @@ class MongoDBDeleteActionTest extends BaseTest // directly insert some mongo records // //////////////////////////////////////// MongoDatabase database = getMongoClient().getDatabase(TestUtils.MONGO_DATABASE); - MongoCollection collection = database.getCollection(TestUtils.TEST_COLLECTION); + MongoCollection collection = database.getCollection(TestUtils.TABLE_NAME_PERSON); collection.insertMany(List.of( Document.parse(""" {"firstName": "Darin", "lastName": "Kelkhoff"}"""), diff --git a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertActionTest.java b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertActionTest.java index 06e8c3cc..10c16f8e 100644 --- a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertActionTest.java +++ b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertActionTest.java @@ -78,7 +78,7 @@ class MongoDBInsertActionTest extends BaseTest // directly query mongo for the inserted records // /////////////////////////////////////////////////// MongoDatabase database = getMongoClient().getDatabase(TestUtils.MONGO_DATABASE); - MongoCollection collection = database.getCollection(TestUtils.TEST_COLLECTION); + MongoCollection collection = database.getCollection(TestUtils.TABLE_NAME_PERSON); assertEquals(3, collection.countDocuments()); for(Document document : collection.find()) { diff --git a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryActionTest.java b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryActionTest.java index 2bb6035c..6afe96ce 100644 --- a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryActionTest.java +++ b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryActionTest.java @@ -23,18 +23,33 @@ package com.kingsrook.qqq.backend.module.mongodb.actions; import java.time.Instant; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.Now; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.module.mongodb.BaseTest; import com.kingsrook.qqq.backend.module.mongodb.TestUtils; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import org.bson.Document; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -51,9 +66,60 @@ class MongoDBQueryActionTest extends BaseTest ** *******************************************************************************/ @BeforeEach - void beforeEach() + void beforeEach() throws QException { + primeTestDatabase(); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void primeTestDatabase() throws QException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON); + insertInput.setRecords(List.of( + new QRecord().withValue("seqNo", 1).withValue("firstName", "Darin").withValue("lastName", "Kelkhoff").withValue("birthDate", LocalDate.parse("1980-05-31")).withValue("email", "darin.kelkhoff@gmail.com").withValue("isEmployed", true).withValue("annualSalary", 25000).withValue("daysWorked", 27).withValue("homeTown", "Chester"), + new QRecord().withValue("seqNo", 2).withValue("firstName", "James").withValue("lastName", "Maes").withValue("birthDate", LocalDate.parse("1980-05-15")).withValue("email", "jmaes@mmltholdings.com").withValue("isEmployed", true).withValue("annualSalary", 26000).withValue("daysWorked", 124).withValue("homeTown", "Chester"), + new QRecord().withValue("seqNo", 3).withValue("firstName", "Tim").withValue("lastName", "Chamberlain").withValue("birthDate", LocalDate.parse("1976-05-28")).withValue("email", "tchamberlain@mmltholdings.com").withValue("isEmployed", false).withValue("annualSalary", null).withValue("daysWorked", 0).withValue("homeTown", "Decatur"), + new QRecord().withValue("seqNo", 4).withValue("firstName", "Tyler").withValue("lastName", "Samples").withValue("birthDate", null).withValue("email", "tsamples@mmltholdings.com").withValue("isEmployed", true).withValue("annualSalary", 30000).withValue("daysWorked", 99).withValue("homeTown", "Texas"), + new QRecord().withValue("seqNo", 5).withValue("firstName", "Garret").withValue("lastName", "Richardson").withValue("birthDate", LocalDate.parse("1981-01-01")).withValue("email", "grichardson@mmltholdings.com").withValue("isEmployed", true).withValue("annualSalary", 1000000).withValue("daysWorked", 232).withValue("homeTown", null) + )); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + + MongoDatabase database = getMongoClient().getDatabase(TestUtils.MONGO_DATABASE); + + MongoCollection storeCollection = database.getCollection(TestUtils.TABLE_NAME_STORE); + storeCollection.insertMany(List.of( + Document.parse(""" + {"key":1, "name": "Q-Mart"}"""), + Document.parse(""" + {"key":2, "name": "QQQ 'R' Us"}"""), + Document.parse(""" + {"key":3, "name": "QDepot"}""") + )); + + MongoCollection orderCollection = database.getCollection(TestUtils.TABLE_NAME_ORDER); + orderCollection.insertMany(List.of( + Document.parse(""" + {"key": 1, "storeKey":1, "billToPersonId": 1, "shipToPersonId": 1}}"""), + Document.parse(""" + {"key": 2, "storeKey":1, "billToPersonId": 1, "shipToPersonId": 2}}"""), + Document.parse(""" + {"key": 3, "storeKey":1, "billToPersonId": 2, "shipToPersonId": 3}}"""), + Document.parse(""" + {"key": 4, "storeKey":2, "billToPersonId": 4, "shipToPersonId": 5}}"""), + Document.parse(""" + {"key": 5, "storeKey":2, "billToPersonId": 5, "shipToPersonId": 4}}"""), + Document.parse(""" + {"key": 6, "storeKey":3, "billToPersonId": 5, "shipToPersonId": null}}"""), + Document.parse(""" + {"key": 7, "storeKey":3, "billToPersonId": null, "shipToPersonId": 5}"""), + Document.parse(""" + {"key": 8, "storeKey":3, "billToPersonId": null, "shipToPersonId": 5}""") + )); } @@ -64,11 +130,16 @@ class MongoDBQueryActionTest extends BaseTest @Test void test() throws QException { + ////////////////////////////////////////////////////////// + // let's not use the primed-database rows for this test // + ////////////////////////////////////////////////////////// + clearDatabase(); + //////////////////////////////////////// // directly insert some mongo records // //////////////////////////////////////// MongoDatabase database = getMongoClient().getDatabase(TestUtils.MONGO_DATABASE); - MongoCollection collection = database.getCollection(TestUtils.TEST_COLLECTION); + MongoCollection collection = database.getCollection(TestUtils.TABLE_NAME_PERSON); collection.insertMany(List.of( Document.parse(""" { "metaData": {"createDate": "2023-01-09T01:01:01.123Z", "modifyDate": "2023-01-09T02:02:02.123Z", "oops": "All Crunchberries"}, @@ -116,4 +187,784 @@ class MongoDBQueryActionTest extends BaseTest assertEquals("Sample", record.getValueString("lastName")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + private QueryInput initQueryRequest() + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_PERSON); + return queryInput; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUnfilteredQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testEqualsQuery() throws QException + { + String email = "darin.kelkhoff@gmail.com"; + + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.EQUALS) + .withValues(List.of(email))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertEquals(email, queryOutput.getRecords().get(0).getValueString("email"), "Should find expected email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotEqualsQuery() throws QException + { + String email = "darin.kelkhoff@gmail.com"; + + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_EQUALS) + .withValues(List.of(email))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").equals(email)), "Should NOT find expected email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotEqualsOrIsNullQuery() throws QException + { + ///////////////////////////////////////////////////////////////////////////// + // 5 rows, 1 has a null salary, 1 has 1,000,000. // + // first confirm that query for != returns 3 (the null does NOT come back) // + // then, confirm that != or is null gives the (more humanly expected) 4. // + ///////////////////////////////////////////////////////////////////////////// + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("annualSalary") + .withOperator(QCriteriaOperator.NOT_EQUALS) + .withValues(List.of(1_000_000)))); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Expected # of rows"); + + queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("annualSalary") + .withOperator(QCriteriaOperator.NOT_EQUALS_OR_IS_NULL) + .withValues(List.of(1_000_000)))); + queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> Objects.equals(1_000_000, r.getValueInteger("annualSalary"))), "Should NOT find expected salary"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("seqNo") + .withOperator(QCriteriaOperator.IN) + .withValues(List.of(2, 4))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(2) || r.getValueInteger("seqNo").equals(4)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotInQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("seqNo") + .withOperator(QCriteriaOperator.NOT_IN) + .withValues(List.of(2, 3, 4))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(1) || r.getValueInteger("seqNo").equals(5)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testStartsWith() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.STARTS_WITH) + .withValues(List.of("darin"))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testContains() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.CONTAINS) + .withValues(List.of("kelkhoff"))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLike() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.LIKE) + .withValues(List.of("%kelk%"))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotLike() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_LIKE) + .withValues(List.of("%kelk%"))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testEndsWith() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.ENDS_WITH) + .withValues(List.of("gmail.com"))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotStartsWith() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_STARTS_WITH) + .withValues(List.of("darin"))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotContains() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_CONTAINS) + .withValues(List.of("kelkhoff"))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotEndsWith() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_ENDS_WITH) + .withValues(List.of("gmail.com"))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLessThanQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("seqNo") + .withOperator(QCriteriaOperator.LESS_THAN) + .withValues(List.of(3))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(1) || r.getValueInteger("seqNo").equals(2)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLessThanOrEqualsQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("seqNo") + .withOperator(QCriteriaOperator.LESS_THAN_OR_EQUALS) + .withValues(List.of(2))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(1) || r.getValueInteger("seqNo").equals(2)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testGreaterThanQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("seqNo") + .withOperator(QCriteriaOperator.GREATER_THAN) + .withValues(List.of(3))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(4) || r.getValueInteger("seqNo").equals(5)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testGreaterThanOrEqualsQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("seqNo") + .withOperator(QCriteriaOperator.GREATER_THAN_OR_EQUALS) + .withValues(List.of(4))) + ); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(4) || r.getValueInteger("seqNo").equals(5)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testIsBlankQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("birthDate") + .withOperator(QCriteriaOperator.IS_BLANK) + )); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("birthDate") == null), "Should find expected row"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testIsNotBlankQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("firstName") + .withOperator(QCriteriaOperator.IS_NOT_BLANK) + )); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("firstName") != null), "Should find expected rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testBetweenQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("seqNo") + .withOperator(QCriteriaOperator.BETWEEN) + .withValues(List.of(2, 4)) + )); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(2) || r.getValueInteger("seqNo").equals(3) || r.getValueInteger("seqNo").equals(4)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + * [ + * And Filter + * { + * filters= + * [ + * Not Filter + * { + * filter=And Filter + * { + * filters= + * [ + * Operator Filter + * { + * fieldName='seqNo', operator='$gte', value=2 + * }, + * Operator Filter + * { + * fieldName='seqNo', operator='$lte', value=4 + * } + * ] + * } + * } + * ] + * } + * ] + *******************************************************************************/ + @Test + public void testNotBetweenQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("seqNo") + .withOperator(QCriteriaOperator.NOT_BETWEEN) + .withValues(List.of(2, 4)) + )); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(1) || r.getValueInteger("seqNo").equals(5)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testFilterExpressions() throws QException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON); + insertInput.setRecords(List.of( + new QRecord().withValue("email", "-").withValue("firstName", "past").withValue("lastName", "ExpressionTest").withValue("birthDate", Instant.now().minus(3, ChronoUnit.DAYS)), + new QRecord().withValue("email", "-").withValue("firstName", "future").withValue("lastName", "ExpressionTest").withValue("birthDate", Instant.now().plus(3, ChronoUnit.DAYS)) + )); + new InsertAction().execute(insertInput); + + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest"))) + .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(new Now())))); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row"); + } + + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest"))) + .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(NowWithOffset.plus(2, ChronoUnit.DAYS))))); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row"); + } + + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest"))) + .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.GREATER_THAN).withValues(List.of(NowWithOffset.minus(5, ChronoUnit.DAYS))))); + QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row"); + Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("future")), "Should find expected row"); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testEmptyInList() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.IN, List.of()))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(0, queryOutput.getRecords().size(), "IN empty list should find nothing."); + + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.NOT_IN, List.of()))); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "NOT_IN empty list should find everything."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOr() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Darin"))) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Tim"))) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "OR should find 2 rows"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNestedFilterAndOrOr() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withSubFilters(List.of( + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("James"))) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Maes"))), + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Darin"))) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Kelkhoff"))) + )) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Complex query should find 2 rows"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("James") && r.getValueString("lastName").equals("Maes")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin") && r.getValueString("lastName").equals("Kelkhoff")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNestedFilterOrAndAnd() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withSubFilters(List.of( + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("James"))) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Tim"))), + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Kelkhoff"))) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Chamberlain"))) + )) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Complex query should find 1 row"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim") && r.getValueString("lastName").equals("Chamberlain")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNestedFilterAndTopLevelFilter() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("seqNo", QCriteriaOperator.EQUALS, 3)) + .withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withSubFilters(List.of( + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("James"))) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Tim"))), + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Kelkhoff"))) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Chamberlain"))) + )) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Complex query should find 1 row"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueInteger("seqNo").equals(3) && r.getValueString("firstName").equals("Tim") && r.getValueString("lastName").equals("Chamberlain")); + + queryInput.getFilter().setCriteria(List.of(new QFilterCriteria("seqNo", QCriteriaOperator.NOT_EQUALS, 3))); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(0, queryOutput.getRecords().size(), "Next complex query should find 0 rows"); + } + + + + /******************************************************************************* + ** queries on the store table, where the primary key (id) is the security field + *******************************************************************************/ + @Test + void testRecordSecurityPrimaryKeyFieldNoFilters() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_STORE); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(3); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(1) + .anyMatch(r -> r.getValueInteger("key").equals(1)); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(1) + .anyMatch(r -> r.getValueInteger("key").equals(2)); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5)); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, null)); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, Collections.emptyList())); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, List.of(1, 3))); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(2) + .anyMatch(r -> r.getValueInteger("key").equals(1)) + .anyMatch(r -> r.getValueInteger("key").equals(3)); + } + + + + /******************************************************************************* + ** not really expected to be any different from where we filter on the primary key, + ** but just good to make sure + *******************************************************************************/ + @Test + void testRecordSecurityForeignKeyFieldNoFilters() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(8); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(3) + .allMatch(r -> r.getValueInteger("storeKey").equals(1)); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(2) + .allMatch(r -> r.getValueInteger("storeKey").equals(2)); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5)); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, null)); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, Collections.emptyList())); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, List.of(1, 3))); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(6) + .allMatch(r -> r.getValueInteger("storeKey").equals(1) || r.getValueInteger("storeKey").equals(3)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurityWithFilters() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("key", QCriteriaOperator.BETWEEN, List.of(2, 7)))); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(6); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("key", QCriteriaOperator.BETWEEN, List.of(2, 7)))); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(2) + .allMatch(r -> r.getValueInteger("storeKey").equals(1)); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("key", QCriteriaOperator.BETWEEN, List.of(2, 7)))); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5)); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("key", QCriteriaOperator.BETWEEN, List.of(2, 7)))); + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("storeKey", QCriteriaOperator.IN, List.of(1, 2)))); + QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, List.of(1, 3))); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(3) + .allMatch(r -> r.getValueInteger("storeKey").equals(1)); + } + } \ No newline at end of file diff --git a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBTransactionTest.java b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBTransactionTest.java new file mode 100644 index 00000000..423703ee --- /dev/null +++ b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBTransactionTest.java @@ -0,0 +1,114 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb.actions; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.mongodb.BaseTest; +import com.kingsrook.qqq.backend.module.mongodb.TestUtils; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; +import com.mongodb.MongoCommandException; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for MongoDBTransaction + *******************************************************************************/ +class MongoDBTransactionTest extends BaseTest +{ + + /******************************************************************************* + ** Our testcontainer only runs a single mongo, so it doesn't support transactions. + ** The Backend built by TestUtils is configured to with transactionsSupported = false + ** make sure things all work like this. + *******************************************************************************/ + @Test + void testWithTransactionsDisabled() throws QException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON); + insertInput.setRecords(List.of(new QRecord().withValue("firstName", "Darin"))); + + QBackendTransaction transaction = QBackendTransaction.openFor(insertInput); + assertNotNull(transaction); + assertThat(transaction).isInstanceOf(MongoDBTransaction.class); + MongoDBTransaction mongoDBTransaction = (MongoDBTransaction) transaction; + assertNotNull(mongoDBTransaction.getMongoClient()); + assertNotNull(mongoDBTransaction.getClientSession()); + + insertInput.setTransaction(transaction); + new InsertAction().execute(insertInput); + transaction.commit(); + } + + + + /******************************************************************************* + ** make sure we throw an error if we do turn on transaction support, but our + ** mongo backend can't handle them + *******************************************************************************/ + @Test + void testWithTransactionsEnabled() throws QException + { + MongoDBBackendMetaData backend = (MongoDBBackendMetaData) QContext.getQInstance().getBackend(TestUtils.DEFAULT_BACKEND_NAME); + + try + { + backend.setTransactionsSupported(true); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON); + insertInput.setRecords(List.of(new QRecord().withValue("firstName", "Darin"))); + + QBackendTransaction transaction = QBackendTransaction.openFor(insertInput); + assertNotNull(transaction); + assertThat(transaction).isInstanceOf(MongoDBTransaction.class); + MongoDBTransaction mongoDBTransaction = (MongoDBTransaction) transaction; + assertNotNull(mongoDBTransaction.getMongoClient()); + assertNotNull(mongoDBTransaction.getClientSession()); + + insertInput.setTransaction(transaction); + + assertThatThrownBy(() -> new InsertAction().execute(insertInput)) + .isInstanceOf(QException.class) + .hasRootCauseInstanceOf(MongoCommandException.class); + + assertThatThrownBy(() -> transaction.commit()) + .isInstanceOf(QException.class) + .hasRootCauseInstanceOf(MongoCommandException.class); + } + finally + { + backend.setTransactionsSupported(false); + } + } + +} \ No newline at end of file diff --git a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateActionTest.java b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateActionTest.java index 6412cd4a..334a08ea 100644 --- a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateActionTest.java +++ b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateActionTest.java @@ -22,9 +22,24 @@ package com.kingsrook.qqq.backend.module.mongodb.actions; +import java.time.Instant; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.module.mongodb.BaseTest; +import com.kingsrook.qqq.backend.module.mongodb.TestUtils; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.result.InsertManyResult; +import org.bson.BsonValue; +import org.bson.Document; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; /******************************************************************************* @@ -39,7 +54,34 @@ class MongoDBUpdateActionTest extends BaseTest @Test void test() throws QException { - // todo - test!! + //////////////////////////////////////// + // directly insert some mongo records // + //////////////////////////////////////// + MongoDatabase database = getMongoClient().getDatabase(TestUtils.MONGO_DATABASE); + MongoCollection collection = database.getCollection(TestUtils.TABLE_NAME_PERSON); + InsertManyResult insertManyResult = collection.insertMany(List.of( + Document.parse(""" + {"metaData": {"createDate": "2023-01-09T03:03:03.123Z", "modifyDate": "2023-01-09T04:04:04.123Z"}, "firstName": "Tylers", "lastName": "Sample"}""") + )); + BsonValue insertedId = insertManyResult.getInsertedIds().values().iterator().next(); + + //////////////////////////////////// + // update using qqq update action // + //////////////////////////////////// + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(TestUtils.TABLE_NAME_PERSON); + updateInput.setRecords(List.of( + new QRecord().withValue("id", insertedId.asObjectId().getValue().toString()).withValue("firstName", "Tyler").withValue("lastName", "Sample") + )); + UpdateOutput updateOutput = new UpdateAction().execute(updateInput); + + ///////////////////////////////////////////////// + // directly query mongo for the updated record // + ///////////////////////////////////////////////// + Document document = collection.find(new Document("firstName", "Tyler")).first(); + assertNotNull(document); + assertEquals("Tyler", document.get("firstName")); + assertNotEquals(Instant.parse("2023-01-09T04:04:04.123Z"), ((Document) document.get("metaData")).get("modifyDate")); } } \ No newline at end of file