diff --git a/pom.xml b/pom.xml index c4e668aa..86a52228 100644 --- a/pom.xml +++ b/pom.xml @@ -33,6 +33,7 @@ qqq-backend-module-api qqq-backend-module-filesystem qqq-backend-module-rdbms + qqq-backend-module-mongodb qqq-language-support-javascript qqq-middleware-picocli qqq-middleware-javalin diff --git a/qqq-backend-module-mongodb/pom.xml b/qqq-backend-module-mongodb/pom.xml new file mode 100644 index 00000000..170ba8a1 --- /dev/null +++ b/qqq-backend-module-mongodb/pom.xml @@ -0,0 +1,120 @@ + + + + + 4.0.0 + + qqq-backend-module-mongodb + + + com.kingsrook.qqq + qqq-parent-project + ${revision} + + + + + + + + + + + com.kingsrook.qqq + qqq-backend-core + ${revision} + + + + + org.mongodb + mongodb-driver-sync + 4.11.1 + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.17.1 + + + + org.testcontainers + mongodb + 1.19.3 + test + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.assertj + assertj-core + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + false + + + *:* + + META-INF/* + + + + + + + ${plugin.shade.phase} + + shade + + + + + + + + \ No newline at end of file diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/MongoDBBackendModule.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/MongoDBBackendModule.java new file mode 100644 index 00000000..cb205654 --- /dev/null +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/MongoDBBackendModule.java @@ -0,0 +1,168 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb; + + +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; +import com.kingsrook.qqq.backend.module.mongodb.actions.AbstractMongoDBAction; +import com.kingsrook.qqq.backend.module.mongodb.actions.MongoClientContainer; +import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBAggregateAction; +import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBCountAction; +import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBDeleteAction; +import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBInsertAction; +import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBQueryAction; +import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBTransaction; +import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBUpdateAction; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBTableBackendDetails; + + +/******************************************************************************* + ** QQQ Backend module for working with MongoDB + *******************************************************************************/ +public class MongoDBBackendModule implements QBackendModuleInterface +{ + static + { + QBackendModuleDispatcher.registerBackendModule(new MongoDBBackendModule()); + } + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + public String getBackendType() + { + return ("mongodb"); + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (MongoDBBackendMetaData.class); + } + + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return (MongoDBTableBackendDetails.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public CountInterface getCountInterface() + { + return (new MongoDBCountAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInterface getQueryInterface() + { + return (new MongoDBQueryAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InsertInterface getInsertInterface() + { + return (new MongoDBInsertAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public UpdateInterface getUpdateInterface() + { + return (new MongoDBUpdateAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public DeleteInterface getDeleteInterface() + { + return (new MongoDBDeleteAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public AggregateInterface getAggregateInterface() + { + return (new MongoDBAggregateAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QBackendTransaction openTransaction(AbstractTableActionInput input) + { + MongoDBBackendMetaData backend = (MongoDBBackendMetaData) input.getBackend(); + MongoClientContainer mongoClientContainer = new AbstractMongoDBAction().openClient(backend, null); + return (new MongoDBTransaction(backend, mongoClientContainer.getMongoClient())); + } +} diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java new file mode 100644 index 00000000..f51704bc --- /dev/null +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java @@ -0,0 +1,541 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb.actions; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.regex.Pattern; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBTableBackendDetails; +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.MongoCredential; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.model.Filters; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.bson.types.ObjectId; + + +/******************************************************************************* + ** Base class for all mongoDB module actions. + *******************************************************************************/ +public class AbstractMongoDBAction +{ + private static final QLogger LOG = QLogger.getLogger(AbstractMongoDBAction.class); + + + + /******************************************************************************* + ** Open a MongoDB Client / session -- re-using the one in the input transaction + ** if it is present. + *******************************************************************************/ + public MongoClientContainer openClient(MongoDBBackendMetaData backend, QBackendTransaction transaction) + { + if(transaction instanceof MongoDBTransaction mongoDBTransaction) + { + ////////////////////////////////////////////////////////////////////////////////////////// + // re-use the connection from the transaction (indicating false in last parameter here) // + ////////////////////////////////////////////////////////////////////////////////////////// + return (new MongoClientContainer(mongoDBTransaction.getMongoClient(), mongoDBTransaction.getClientSession(), false)); + } + + ConnectionString connectionString = new ConnectionString("mongodb://" + backend.getHost() + ":" + backend.getPort() + "/"); + + MongoCredential credential = MongoCredential.createCredential(backend.getUsername(), backend.getAuthSourceDatabase(), backend.getPassword().toCharArray()); + + MongoClientSettings settings = MongoClientSettings.builder() + + //////////////////////////////////////////////// + // is this needed, what, for a cluster maybe? // + //////////////////////////////////////////////// + // .applyToClusterSettings(builder -> builder.hosts(seeds)) + + .applyConnectionString(connectionString) + .credential(credential) + .build(); + + MongoClient mongoClient = MongoClients.create(settings); + + //////////////////////////////////////////////////////////////////////////// + // indicate that this connection was newly opened via the true param here // + //////////////////////////////////////////////////////////////////////////// + return (new MongoClientContainer(mongoClient, mongoClient.startSession(), true)); + } + + + + /******************************************************************************* + ** Get the name to use for a field in the mongoDB, from the fieldMetaData. + ** + ** That is, field.backendName if set -- else, field.name + *******************************************************************************/ + protected String getFieldBackendName(QFieldMetaData field) + { + if(field.getBackendName() != null) + { + return (field.getBackendName()); + } + return (field.getName()); + } + + + + /******************************************************************************* + ** Get the name to use for a table in the mongoDB, from the table's backendDetails. + ** + ** else, the table's name. + *******************************************************************************/ + protected String getBackendTableName(QTableMetaData table) + { + if(table.getBackendDetails() != null) + { + String backendTableName = ((MongoDBTableBackendDetails) table.getBackendDetails()).getTableName(); + if(StringUtils.hasContent(backendTableName)) + { + return (backendTableName); + } + } + return table.getName(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected int getPageSize() + { + return (1000); + } + + + + /******************************************************************************* + ** Convert a mongodb document to a QRecord. + *******************************************************************************/ + protected QRecord documentToRecord(QTableMetaData table, Document document) + { + QRecord record = new QRecord(); + record.setTableName(table.getName()); + + /////////////////////////////////////////////////////////////////////////// + // todo - this - or iterate over the values in the document?? // + // seems like, maybe, this is an attribute in the table-backend-details? // + /////////////////////////////////////////////////////////////////////////// + Map values = record.getValues(); + for(QFieldMetaData field : table.getFields().values()) + { + String fieldBackendName = getFieldBackendName(field); + Object value = document.get(fieldBackendName); + String fieldName = field.getName(); + + setValue(values, fieldName, value); + } + return (record); + } + + + + /******************************************************************************* + ** Recursive helper method to put a value in a map - where mongodb documents + ** are recursively expanded, and types are mapped to QQQ expectations. + *******************************************************************************/ + private void setValue(Map values, String fieldName, Object value) + { + if(value instanceof ObjectId objectId) + { + values.put(fieldName, objectId.toString()); + } + else if(value instanceof java.util.Date date) + { + values.put(fieldName, date.toInstant()); + } + else if(value instanceof Document document) + { + LinkedHashMap subValues = new LinkedHashMap<>(); + values.put(fieldName, subValues); + + for(String subFieldName : document.keySet()) + { + Object subValue = document.get(subFieldName); + setValue(subValues, subFieldName, subValue); + } + } + else if(value instanceof Serializable s) + { + values.put(fieldName, s); + } + else if(value != null) + { + values.put(fieldName, String.valueOf(value)); + } + else + { + values.put(fieldName, null); + } + } + + + + /******************************************************************************* + ** Convert a QRecord to a mongodb document. + *******************************************************************************/ + protected Document recordToDocument(QTableMetaData table, QRecord record) + { + Document document = new Document(); + + /////////////////////////////////////////////////////////////////////////// + // todo - this - or iterate over the values in the record?? // + // seems like, maybe, this is an attribute in the table-backend-details? // + /////////////////////////////////////////////////////////////////////////// + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getName().equals(table.getPrimaryKeyField()) && record.getValue(field.getName()) == null) + { + //////////////////////////////////// + // let mongodb client generate id // + //////////////////////////////////// + continue; + } + + String fieldBackendName = getFieldBackendName(field); + document.append(fieldBackendName, record.getValue(field.getName())); + } + return (document); + } + + + + /******************************************************************************* + ** Convert QQueryFilter to Bson search query document - including security + ** for the table if needed. + *******************************************************************************/ + protected Bson makeSearchQueryDocument(QTableMetaData table, QQueryFilter filter) throws QException + { + Bson searchQueryWithoutSecurity = makeSearchQueryDocumentWithoutSecurity(table, filter); + QQueryFilter securityFilter = makeSecurityQueryFilter(table); + if(!securityFilter.hasAnyCriteria()) + { + return (searchQueryWithoutSecurity); + } + + Bson searchQueryForSecurity = makeSearchQueryDocumentWithoutSecurity(table, securityFilter); + return (Filters.and(searchQueryWithoutSecurity, searchQueryForSecurity)); + } + + + + /******************************************************************************* + ** Build a QQueryFilter to apply record-level security to the query. + ** Note, it may be empty, if there are no lock fields, or all are all-access. + ** + ** Originally copied from RDBMS module... should this be shared? + ** and/or, how big of a re-write did that get in the joins-enhancements branch... + *******************************************************************************/ + private QQueryFilter makeSecurityQueryFilter(QTableMetaData table) throws QException + { + QQueryFilter securityFilter = new QQueryFilter(); + securityFilter.setBooleanOperator(QQueryFilter.BooleanOperator.AND); + + for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks()))) + { + addSubFilterForRecordSecurityLock(QContext.getQInstance(), QContext.getQSession(), table, securityFilter, recordSecurityLock, null, table.getName(), false); + } + + return (securityFilter); + } + + + + /******************************************************************************* + ** Helper for makeSecuritySearchQuery. + ** + ** Originally copied from RDBMS module... should this be shared? + ** and/or, how big of a re-write did that get in the joins-enhancements branch... + *******************************************************************************/ + private static void addSubFilterForRecordSecurityLock(QInstance instance, QSession session, QTableMetaData table, QQueryFilter securityFilter, RecordSecurityLock recordSecurityLock, JoinsContext joinsContext, String tableNameOrAlias, boolean isOuter) throws QException + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // check if the key type has an all-access key, and if so, if it's set to true for the current user/session // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType()); + if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName())) + { + if(session.hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN)) + { + /////////////////////////////////////////////////////////////////////////////// + // if we have all-access on this key, then we don't need a criterion for it. // + /////////////////////////////////////////////////////////////////////////////// + return; + } + } + + /////////////////////////////////////////////////////////////////////////////////////// + // some differences from RDBMS here, due to not yet having joins support in mongo... // + /////////////////////////////////////////////////////////////////////////////////////// + // String fieldName = tableNameOrAlias + "." + recordSecurityLock.getFieldName(); + String fieldName = recordSecurityLock.getFieldName(); + if(CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain())) + { + throw (new QException("Security locks in mongodb with joinNameChain is not yet supported")); + // fieldName = recordSecurityLock.getFieldName(); + } + + /////////////////////////////////////////////////////////////////////////////////////////// + // else - get the key values from the session and decide what kind of criterion to build // + /////////////////////////////////////////////////////////////////////////////////////////// + QQueryFilter lockFilter = new QQueryFilter(); + List lockCriteria = new ArrayList<>(); + lockFilter.setCriteria(lockCriteria); + + QFieldType type = QFieldType.INTEGER; + try + { + if(joinsContext == null) + { + type = table.getField(fieldName).getType(); + } + else + { + JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(fieldName); + type = fieldAndTableNameOrAlias.field().getType(); + } + } + catch(Exception e) + { + LOG.debug("Error getting field type... Trying Integer", e); + } + + List securityKeyValues = session.getSecurityKeyValues(recordSecurityLock.getSecurityKeyType(), type); + if(CollectionUtils.nullSafeIsEmpty(securityKeyValues)) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // handle user with no values -- they can only see null values, and only iff the lock's null-value behavior is ALLOW // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(RecordSecurityLock.NullValueBehavior.ALLOW.equals(recordSecurityLock.getNullValueBehavior())) + { + lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK)); + } + else + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else, if no user/session values, and null-value behavior is deny, then setup a FALSE condition, to allow no rows. // + // todo - make some explicit contradiction here - maybe even avoid running the whole query - as you're not allowed ANY records // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IN, Collections.emptyList())); + } + } + else + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else, if user/session has some values, build an IN rule - // + // noting that if the lock's null-value behavior is ALLOW, then we actually want IS_NULL_OR_IN, not just IN // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(RecordSecurityLock.NullValueBehavior.ALLOW.equals(recordSecurityLock.getNullValueBehavior())) + { + lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IS_NULL_OR_IN, securityKeyValues)); + } + else + { + lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IN, securityKeyValues)); + } + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if this field is on the outer side of an outer join, then if we do a straight filter on it, then we're basically // + // nullifying the outer join... so for an outer join use-case, OR the security field criteria with a primary-key IS NULL // + // which will make missing rows from the join be found. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(isOuter) + { + lockFilter.setBooleanOperator(QQueryFilter.BooleanOperator.OR); + lockFilter.addCriteria(new QFilterCriteria(tableNameOrAlias + "." + table.getPrimaryKeyField(), QCriteriaOperator.IS_BLANK)); + } + + securityFilter.addSubFilter(lockFilter); + } + + + + /******************************************************************************* + ** w/o considering security, just map a QQueryFilter to a Bson searchQuery. + *******************************************************************************/ + @SuppressWarnings("checkstyle:Indentation") + private Bson makeSearchQueryDocumentWithoutSecurity(QTableMetaData table, QQueryFilter filter) + { + if(filter == null || !filter.hasAnyCriteria()) + { + return (new Document()); + } + + List criteriaFilters = new ArrayList<>(); + + for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria())) + { + List values = criteria.getValues() == null ? new ArrayList<>() : new ArrayList<>(criteria.getValues()); + QFieldMetaData field = table.getField(criteria.getFieldName()); + String fieldBackendName = getFieldBackendName(field); + + if(field.getName().equals(table.getPrimaryKeyField())) + { + ListIterator iterator = values.listIterator(); + while(iterator.hasNext()) + { + Serializable value = iterator.next(); + iterator.set(new ObjectId(String.valueOf(value))); + } + } + + Serializable value0 = values.get(0); + criteriaFilters.add(switch(criteria.getOperator()) + { + case EQUALS -> Filters.eq(fieldBackendName, value0); + case NOT_EQUALS -> Filters.ne(fieldBackendName, value0); + case NOT_EQUALS_OR_IS_NULL -> Filters.or( + Filters.eq(fieldBackendName, null), + Filters.ne(fieldBackendName, value0) + ); + case IN -> filterIn(fieldBackendName, values); + case NOT_IN -> Filters.not(filterIn(fieldBackendName, values)); + case IS_NULL_OR_IN -> Filters.or( + Filters.eq(fieldBackendName, null), + filterIn(fieldBackendName, values) + ); + case LIKE -> filterRegex(fieldBackendName, null, ValueUtils.getValueAsString(value0).replaceAll("%", ".*"), null); + case NOT_LIKE -> Filters.not(filterRegex(fieldBackendName, null, ValueUtils.getValueAsString(value0).replaceAll("%", ".*"), null)); + case STARTS_WITH -> filterRegex(fieldBackendName, null, value0, ".*"); + case ENDS_WITH -> filterRegex(fieldBackendName, ".*", value0, null); + case CONTAINS -> filterRegex(fieldBackendName, ".*", value0, ".*"); + case NOT_STARTS_WITH -> Filters.not(filterRegex(fieldBackendName, null, value0, ".*")); + case NOT_ENDS_WITH -> Filters.not(filterRegex(fieldBackendName, ".*", value0, null)); + case NOT_CONTAINS -> Filters.not(filterRegex(fieldBackendName, ".*", value0, ".*")); + case LESS_THAN -> Filters.lt(fieldBackendName, value0); + case LESS_THAN_OR_EQUALS -> Filters.lte(fieldBackendName, value0); + case GREATER_THAN -> Filters.gt(fieldBackendName, value0); + case GREATER_THAN_OR_EQUALS -> Filters.gte(fieldBackendName, value0); + case IS_BLANK -> filterIsBlank(fieldBackendName); + case IS_NOT_BLANK -> Filters.not(filterIsBlank(fieldBackendName)); + case BETWEEN -> filterBetween(fieldBackendName, values); + case NOT_BETWEEN -> Filters.not(filterBetween(fieldBackendName, values)); + }); + } + + ///////////////////////////////////// + // recursively process sub-filters // + ///////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(filter.getSubFilters())) + { + for(QQueryFilter subFilter : filter.getSubFilters()) + { + criteriaFilters.add(makeSearchQueryDocumentWithoutSecurity(table, subFilter)); + } + } + + Bson bson = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? Filters.and(criteriaFilters) : Filters.or(criteriaFilters); + return bson; + } + + + + /******************************************************************************* + ** build a bson filter doing a regex (e.g., for LIKE, STARTS_WITH, etc) + *******************************************************************************/ + private Bson filterRegex(String fieldBackendName, String prefix, Serializable mainRegex, String suffix) + { + if(prefix == null) + { + prefix = ""; + } + + if(suffix == null) + { + suffix = ""; + } + + String fullRegex = prefix + Pattern.quote(ValueUtils.getValueAsString(mainRegex) + suffix); + return (Filters.regex(fieldBackendName, Pattern.compile(fullRegex))); + } + + + + /******************************************************************************* + ** build a bson filter doing IN + *******************************************************************************/ + private static Bson filterIn(String fieldBackendName, List values) + { + return Filters.in(fieldBackendName, values); + } + + + + /******************************************************************************* + ** build a bson filter doing BETWEEN + *******************************************************************************/ + private static Bson filterBetween(String fieldBackendName, List values) + { + return Filters.and( + Filters.gte(fieldBackendName, values.get(0)), + Filters.lte(fieldBackendName, values.get(1)) + ); + } + + + + /******************************************************************************* + ** build a bson filter doing BLANK (null or == "") + *******************************************************************************/ + private static Bson filterIsBlank(String fieldBackendName) + { + return Filters.or( + Filters.eq(fieldBackendName, null), + Filters.eq(fieldBackendName, "") + ); + } +} diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoClientContainer.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoClientContainer.java new file mode 100644 index 00000000..61289a46 --- /dev/null +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoClientContainer.java @@ -0,0 +1,158 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb.actions; + + +import com.mongodb.client.ClientSession; +import com.mongodb.client.MongoClient; + + +/******************************************************************************* + ** Wrapper around a MongoClient, ClientSession, and a boolean to help signal + ** where it was opened (e.g., so you know if you need to close it yourself, or + ** if it came from someone else (e.g., via an input transaction)). + *******************************************************************************/ +public class MongoClientContainer +{ + private MongoClient mongoClient; + private ClientSession mongoSession; + private boolean needToClose; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public MongoClientContainer(MongoClient mongoClient, ClientSession mongoSession, boolean needToClose) + { + this.mongoClient = mongoClient; + this.mongoSession = mongoSession; + this.needToClose = needToClose; + } + + + + /******************************************************************************* + ** Getter for mongoClient + *******************************************************************************/ + public MongoClient getMongoClient() + { + return (this.mongoClient); + } + + + + /******************************************************************************* + ** Setter for mongoClient + *******************************************************************************/ + public void setMongoClient(MongoClient mongoClient) + { + this.mongoClient = mongoClient; + } + + + + /******************************************************************************* + ** Fluent setter for mongoClient + *******************************************************************************/ + public MongoClientContainer withMongoClient(MongoClient mongoClient) + { + this.mongoClient = mongoClient; + return (this); + } + + + + /******************************************************************************* + ** Getter for mongoSession + *******************************************************************************/ + public ClientSession getMongoSession() + { + return (this.mongoSession); + } + + + + /******************************************************************************* + ** Setter for mongoSession + *******************************************************************************/ + public void setMongoSession(ClientSession mongoSession) + { + this.mongoSession = mongoSession; + } + + + + /******************************************************************************* + ** Fluent setter for mongoSession + *******************************************************************************/ + public MongoClientContainer withMongoSession(ClientSession mongoSession) + { + this.mongoSession = mongoSession; + return (this); + } + + + + /******************************************************************************* + ** Getter for needToClose + *******************************************************************************/ + public boolean getNeedToClose() + { + return (this.needToClose); + } + + + + /******************************************************************************* + ** Setter for needToClose + *******************************************************************************/ + public void setNeedToClose(boolean needToClose) + { + this.needToClose = needToClose; + } + + + + /******************************************************************************* + ** Fluent setter for needToClose + *******************************************************************************/ + public MongoClientContainer withNeedToClose(boolean needToClose) + { + this.needToClose = needToClose; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void closeIfNeeded() + { + if(needToClose) + { + mongoSession.close(); + mongoClient.close(); + } + } +} diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateAction.java new file mode 100644 index 00000000..60b34fde --- /dev/null +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateAction.java @@ -0,0 +1,251 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb.actions; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Accumulators; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.BsonField; +import org.bson.Document; +import org.bson.conversions.Bson; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class MongoDBAggregateAction extends AbstractMongoDBAction implements AggregateInterface +{ + private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class); + + // todo? private ActionTimeoutHelper actionTimeoutHelper; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("checkstyle:indentation") + public AggregateOutput execute(AggregateInput aggregateInput) throws QException + { + MongoClientContainer mongoClientContainer = null; + + try + { + AggregateOutput aggregateOutput = new AggregateOutput(); + QTableMetaData table = aggregateInput.getTable(); + String backendTableName = getBackendTableName(table); + MongoDBBackendMetaData backend = (MongoDBBackendMetaData) aggregateInput.getBackend(); + + mongoClientContainer = openClient(backend, null); // todo - aggregate input has no transaction!? + MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName()); + MongoCollection collection = database.getCollection(backendTableName); + + QQueryFilter filter = aggregateInput.getFilter(); + Bson searchQuery = makeSearchQueryDocument(table, filter); + + ///////////////////////////////////////////////////////////////////////// + // we have to submit a list of BSON objects to the aggregate function. // + // the first one is the search query // + // second is the group-by stuff, which we'll explain as we build it // + ///////////////////////////////////////////////////////////////////////// + List bsonList = new ArrayList<>(); + bsonList.add(Aggregates.match(searchQuery)); + + ////////////////////////////////////////////////////////////////////////////////////// + // if there are group-by fields, then we need to build a document with those fields // + // not sure what the whole name, $name is, but, go go mongo // + ////////////////////////////////////////////////////////////////////////////////////// + Document groupValueDocument = new Document(); + if(CollectionUtils.nullSafeHasContents(aggregateInput.getGroupBys())) + { + for(GroupBy groupBy : aggregateInput.getGroupBys()) + { + String name = getFieldBackendName(table.getField(groupBy.getFieldName())); + groupValueDocument.append(name, "$" + name); + } + } + + //////////////////////////////////////////////////////////////////// + // next build a list of accumulator fields - for aggregate values // + //////////////////////////////////////////////////////////////////// + List bsonFields = new ArrayList<>(); + for(Aggregate aggregate : aggregateInput.getAggregates()) + { + String fieldName = aggregate.getFieldName() + "_" + aggregate.getOperator().toString().toLowerCase(); + String expression = "$" + getFieldBackendName(table.getField(aggregate.getFieldName())); + + bsonFields.add(switch(aggregate.getOperator()) + { + case COUNT -> Accumulators.sum(fieldName, 1); // count... do a sum of 1's + case COUNT_DISTINCT -> throw new QException("Count Distinct is not supported for MongoDB tables at this time."); + case SUM -> Accumulators.sum(fieldName, expression); + case MIN -> Accumulators.min(fieldName, expression); + case MAX -> Accumulators.max(fieldName, expression); + case AVG -> Accumulators.avg(fieldName, expression); + }); + } + + /////////////////////////////////////////////////////////////////////////////////// + // add the group-by fields and the aggregates in the group stage of the pipeline // + /////////////////////////////////////////////////////////////////////////////////// + bsonList.add(Aggregates.group(groupValueDocument, bsonFields)); + + ////////////////////////////////////////////// + // if there are any order-bys, add them too // + ////////////////////////////////////////////// + if(filter != null && CollectionUtils.nullSafeHasContents(filter.getOrderBys())) + { + Document sortValue = new Document(); + for(QFilterOrderBy orderBy : filter.getOrderBys()) + { + String fieldName; + if(orderBy instanceof QFilterOrderByAggregate orderByAggregate) + { + Aggregate aggregate = orderByAggregate.getAggregate(); + fieldName = aggregate.getFieldName() + "_" + aggregate.getOperator().toString().toLowerCase(); + } + else if(orderBy instanceof QFilterOrderByGroupBy orderByGroupBy) + { + fieldName = "_id." + getFieldBackendName(table.getField(orderByGroupBy.getGroupBy().getFieldName())); + } + else + { + /////////////////////////////////////////////////// + // does this happen? should it be "_id." if so? // + /////////////////////////////////////////////////// + fieldName = getFieldBackendName(table.getField(orderBy.getFieldName())); + } + + sortValue.append(fieldName, orderBy.getIsAscending() ? 1 : -1); + } + + bsonList.add(new Document("$sort", sortValue)); + } + + //////////////////////////////////////////////////////// + // todo - system property to control (like print-sql) // + //////////////////////////////////////////////////////// + // LOG.debug(bsonList.toString()); + + /////////////////////////// + // execute the aggregate // + /////////////////////////// + AggregateIterable aggregates = collection.aggregate(mongoClientContainer.getMongoSession(), bsonList); + + List results = new ArrayList<>(); + aggregateOutput.setResults(results); + + ///////////////////// + // process results // + ///////////////////// + for(Document document : aggregates) + { + AggregateResult result = new AggregateResult(); + results.add(result); + + //////////////////////////////////////////////////////////////// + // get group by values (if there are any) out of the document // + //////////////////////////////////////////////////////////////// + for(GroupBy groupBy : CollectionUtils.nonNullList(aggregateInput.getGroupBys())) + { + Document idDocument = (Document) document.get("_id"); + Object value = idDocument.get(groupBy.getFieldName()); + result.withGroupByValue(groupBy, ValueUtils.getValueAsFieldType(groupBy.getType(), value)); + } + + ////////////////////////////////////////// + // get aggregate values out of document // + ////////////////////////////////////////// + for(Aggregate aggregate : aggregateInput.getAggregates()) + { + QFieldMetaData field = table.getField(aggregate.getFieldName()); + QFieldType fieldType = aggregate.getFieldType(); + if(fieldType == null) + { + fieldType = field.getType(); + } + if(fieldType.equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG))) + { + fieldType = QFieldType.DECIMAL; + } + + Object value = document.get(aggregate.getFieldName() + "_" + aggregate.getOperator().toString().toLowerCase()); + result.withAggregateValue(aggregate, ValueUtils.getValueAsFieldType(fieldType, value)); + } + } + + return (aggregateOutput); + } + catch(Exception e) + { + /* + if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout()) + { + setCountStatFirstResultTime(); + throw (new QUserFacingException("Aggregate timed out.")); + } + + if(isCancelled) + { + throw (new QUserFacingException("Aggregate was cancelled.")); + } + */ + + LOG.warn("Error executing aggregate", e); + throw new QException("Error executing aggregate", e); + } + finally + + { + if(mongoClientContainer != null) + { + mongoClientContainer.closeIfNeeded(); + } + } + } + +} diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountAction.java new file mode 100644 index 00000000..277977a7 --- /dev/null +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountAction.java @@ -0,0 +1,119 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb.actions; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Accumulators; +import com.mongodb.client.model.Aggregates; +import org.bson.Document; +import org.bson.conversions.Bson; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class MongoDBCountAction extends AbstractMongoDBAction implements CountInterface +{ + private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class); + + // todo? private ActionTimeoutHelper actionTimeoutHelper; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public CountOutput execute(CountInput countInput) throws QException + { + MongoClientContainer mongoClientContainer = null; + + try + { + CountOutput countOutput = new CountOutput(); + QTableMetaData table = countInput.getTable(); + String backendTableName = getBackendTableName(table); + MongoDBBackendMetaData backend = (MongoDBBackendMetaData) countInput.getBackend(); + + mongoClientContainer = openClient(backend, null); // todo - count input has no transaction!? + MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName()); + MongoCollection collection = database.getCollection(backendTableName); + + QQueryFilter filter = countInput.getFilter(); + Bson searchQuery = makeSearchQueryDocument(table, filter); + + List bsonList = List.of( + Aggregates.match(searchQuery), + Aggregates.group("_id", Accumulators.sum("count", 1))); + + //////////////////////////////////////////////////////// + // todo - system property to control (like print-sql) // + //////////////////////////////////////////////////////// + // LOG.debug(bsonList.toString()); + + AggregateIterable aggregate = collection.aggregate(mongoClientContainer.getMongoSession(), bsonList); + + Document document = aggregate.first(); + countOutput.setCount(document == null ? 0 : document.get("count", Integer.class)); + + return (countOutput); + } + catch(Exception e) + { + /* + if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout()) + { + setCountStatFirstResultTime(); + throw (new QUserFacingException("Count timed out.")); + } + + if(isCancelled) + { + throw (new QUserFacingException("Count was cancelled.")); + } + */ + + LOG.warn("Error executing count", e); + throw new QException("Error executing count", e); + } + finally + { + if(mongoClientContainer != null) + { + mongoClientContainer.closeIfNeeded(); + } + } + } + +} diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteAction.java new file mode 100644 index 00000000..54806601 --- /dev/null +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteAction.java @@ -0,0 +1,129 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Filters; +import com.mongodb.client.result.DeleteResult; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.bson.types.ObjectId; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class MongoDBDeleteAction extends AbstractMongoDBAction implements DeleteInterface +{ + private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public boolean supportsQueryFilterInput() + { + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public DeleteOutput execute(DeleteInput deleteInput) throws QException + { + MongoClientContainer mongoClientContainer = null; + + try + { + DeleteOutput deleteOutput = new DeleteOutput(); + QTableMetaData table = deleteInput.getTable(); + String backendTableName = getBackendTableName(table); + MongoDBBackendMetaData backend = (MongoDBBackendMetaData) deleteInput.getBackend(); + + mongoClientContainer = openClient(backend, deleteInput.getTransaction()); + MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName()); + MongoCollection collection = database.getCollection(backendTableName); + + QQueryFilter queryFilter = deleteInput.getQueryFilter(); + Bson searchQuery; + if(CollectionUtils.nullSafeHasContents(deleteInput.getPrimaryKeys())) + { + searchQuery = Filters.in("_id", deleteInput.getPrimaryKeys().stream().map(id -> new ObjectId(ValueUtils.getValueAsString(id))).toList()); + } + else if(queryFilter != null && queryFilter.hasAnyCriteria()) + { + QQueryFilter filter = queryFilter; + searchQuery = makeSearchQueryDocument(table, filter); + } + else + { + LOG.info("Missing both primary keys and a search filter in delete request - exiting with noop", logPair("tableName", table.getName())); + return (deleteOutput); + } + + //////////////////////////////////////////////////////// + // todo - system property to control (like print-sql) // + //////////////////////////////////////////////////////// + // LOG.debug(searchQuery); + + DeleteResult deleteResult = collection.deleteMany(mongoClientContainer.getMongoSession(), searchQuery); + deleteOutput.setDeletedRecordCount((int) deleteResult.getDeletedCount()); + + ////////////////////////////////////////////////////////////////////////// + // todo any way to get records with errors or warnings for deleteOutput // + ////////////////////////////////////////////////////////////////////////// + + return (deleteOutput); + } + catch(Exception e) + { + LOG.warn("Error executing delete", e); + throw new QException("Error executing delete", e); + } + finally + { + if(mongoClientContainer != null) + { + mongoClientContainer.closeIfNeeded(); + } + } + } + +} diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertAction.java new file mode 100644 index 00000000..fe89411f --- /dev/null +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertAction.java @@ -0,0 +1,266 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb.actions; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.result.InsertManyResult; +import org.bson.BsonValue; +import org.bson.Document; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class MongoDBInsertAction extends AbstractMongoDBAction implements InsertInterface +{ + private static final QLogger LOG = QLogger.getLogger(MongoDBInsertAction.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public InsertOutput execute(InsertInput insertInput) throws QException + { + MongoClientContainer mongoClientContainer = null; + InsertOutput rs = new InsertOutput(); + List outputRecords = new ArrayList<>(); + rs.setRecords(outputRecords); + + try + { + QTableMetaData table = insertInput.getTable(); + String backendTableName = getBackendTableName(table); + MongoDBBackendMetaData backend = (MongoDBBackendMetaData) insertInput.getBackend(); + + mongoClientContainer = openClient(backend, insertInput.getTransaction()); + MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName()); + MongoCollection collection = database.getCollection(backendTableName); + + ////////////////////////// + // todo - transaction?! // + ////////////////////////// + + /////////////////////////////////////////////////////////////////////////// + // page over input record list (assuming some size of batch is too big?) // + /////////////////////////////////////////////////////////////////////////// + for(List page : CollectionUtils.getPages(insertInput.getRecords(), getPageSize())) + { + ////////////////////////////////////////////////////////////////// + // build list of documents from records w/o errors in this page // + ////////////////////////////////////////////////////////////////// + List documentList = new ArrayList<>(); + for(QRecord record : page) + { + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + continue; + } + documentList.add(recordToDocument(table, record)); + } + + ///////////////////////////////////// + // skip pages that were all errors // + ///////////////////////////////////// + if(documentList.isEmpty()) + { + continue; + } + + //////////////////////////////////////////////////////// + // todo - system property to control (like print-sql) // + //////////////////////////////////////////////////////// + // LOG.debug(documentList); + + /////////////////////////////////////////////// + // actually do the insert // + // todo - how are errors returned by mongo?? // + /////////////////////////////////////////////// + InsertManyResult insertManyResult = collection.insertMany(mongoClientContainer.getMongoSession(), documentList); + + ///////////////////////////////// + // put ids on inserted records // + ///////////////////////////////// + int index = 0; + for(QRecord record : page) + { + QRecord outputRecord = new QRecord(record); + rs.addRecord(outputRecord); + + if(CollectionUtils.nullSafeIsEmpty(record.getErrors())) + { + BsonValue insertedId = insertManyResult.getInsertedIds().get(index++); + String idString = insertedId.asObjectId().getValue().toString(); + outputRecord.setValue(table.getPrimaryKeyField(), idString); + } + } + } + } + catch(Exception e) + { + throw new QException("Error executing insert: " + e.getMessage(), e); + } + finally + { + if(mongoClientContainer != null) + { + mongoClientContainer.closeIfNeeded(); + } + } + + return (rs); + + /* + try + { + List insertableFields = table.getFields().values().stream() + .filter(field -> !field.getName().equals("id")) // todo - intent here is to avoid non-insertable fields. + .toList(); + + String columns = insertableFields.stream() + .map(f -> "`" + getColumnName(f) + "`") + .collect(Collectors.joining(", ")); + String questionMarks = insertableFields.stream() + .map(x -> "?") + .collect(Collectors.joining(", ")); + + List outputRecords = new ArrayList<>(); + rs.setRecords(outputRecords); + + Connection connection; + boolean needToCloseConnection = false; + if(insertInput.getTransaction() != null && insertInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction) + { + connection = rdbmsTransaction.getConnection(); + } + else + { + connection = getConnection(insertInput); + needToCloseConnection = true; + } + + try + { + for(List page : CollectionUtils.getPages(insertInput.getRecords(), QueryManager.PAGE_SIZE)) + { + String tableName = escapeIdentifier(getTableName(table)); + StringBuilder sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(columns).append(") VALUES"); + List params = new ArrayList<>(); + int recordIndex = 0; + + ////////////////////////////////////////////////////// + // for each record in the page: // + // - if it has errors, skip it // + // - else add a "(?,?,...,?)," clause to the INSERT // + // - then add all fields into the params list // + ////////////////////////////////////////////////////// + for(QRecord record : page) + { + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + continue; + } + + if(recordIndex++ > 0) + { + sql.append(","); + } + sql.append("(").append(questionMarks).append(")"); + + for(QFieldMetaData field : insertableFields) + { + Serializable value = record.getValue(field.getName()); + value = scrubValue(field, value); + params.add(value); + } + } + + //////////////////////////////////////////////////////////////////////////////////////// + // if all records had errors, copy them to the output, and continue w/o running query // + //////////////////////////////////////////////////////////////////////////////////////// + if(recordIndex == 0) + { + for(QRecord record : page) + { + QRecord outputRecord = new QRecord(record); + outputRecords.add(outputRecord); + } + continue; + } + + Long mark = System.currentTimeMillis(); + + /////////////////////////////////////////////////////////// + // execute the insert, then foreach record in the input, // + // add it to the output, and set its generated id too. // + /////////////////////////////////////////////////////////// + // todo sql customization - can edit sql and/or param list + // todo - non-serial-id style tables + // todo - other generated values, e.g., createDate... maybe need to re-select? + List idList = QueryManager.executeInsertForGeneratedIds(connection, sql.toString(), params); + int index = 0; + for(QRecord record : page) + { + QRecord outputRecord = new QRecord(record); + outputRecords.add(outputRecord); + + if(CollectionUtils.nullSafeIsEmpty(record.getErrors())) + { + Integer id = idList.get(index++); + outputRecord.setValue(table.getPrimaryKeyField(), id); + } + } + + logSQL(sql, params, mark); + } + } + finally + { + if(needToCloseConnection) + { + connection.close(); + } + } + + return rs; + } + catch(Exception e) + { + throw new QException("Error executing insert: " + e.getMessage(), e); + } + */ + } + +} diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryAction.java new file mode 100644 index 00000000..25b433df --- /dev/null +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryAction.java @@ -0,0 +1,163 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import org.bson.Document; +import org.bson.conversions.Bson; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryInterface +{ + private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class); + + // todo? private ActionTimeoutHelper actionTimeoutHelper; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryOutput execute(QueryInput queryInput) throws QException + { + MongoClientContainer mongoClientContainer = null; + + try + { + QueryOutput queryOutput = new QueryOutput(queryInput); + QTableMetaData table = queryInput.getTable(); + String backendTableName = getBackendTableName(table); + MongoDBBackendMetaData backend = (MongoDBBackendMetaData) queryInput.getBackend(); + + mongoClientContainer = openClient(backend, queryInput.getTransaction()); + MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName()); + MongoCollection collection = database.getCollection(backendTableName); + + ///////////////////////// + // set up filter/query // + ///////////////////////// + QQueryFilter filter = queryInput.getFilter(); + Bson searchQuery = makeSearchQueryDocument(table, filter); + + //////////////////////////////////////////////////////// + // todo - system property to control (like print-sql) // + //////////////////////////////////////////////////////// + // LOG.debug(searchQuery); + + //////////////////////////////////////////////////////////// + // create cursor - further adjustments to it still follow // + //////////////////////////////////////////////////////////// + FindIterable cursor = collection.find(mongoClientContainer.getMongoSession(), searchQuery); + + /////////////////////////////////// + // add a sort operator if needed // + /////////////////////////////////// + if(filter != null && CollectionUtils.nullSafeHasContents(filter.getOrderBys())) + { + Document sortDocument = new Document(); + for(QFilterOrderBy orderBy : filter.getOrderBys()) + { + String fieldBackendName = getFieldBackendName(table.getField(orderBy.getFieldName())); + sortDocument.put(fieldBackendName, orderBy.getIsAscending() ? 1 : -1); + } + cursor.sort(sortDocument); + } + + //////////////////////// + // apply skip & limit // + //////////////////////// + if(filter != null) + { + if(filter.getSkip() != null) + { + cursor.skip(filter.getSkip()); + } + + if(filter.getLimit() != null) + { + cursor.limit(filter.getLimit()); + } + } + + //////////////////////////////////////////// + // iterate over results, building records // + //////////////////////////////////////////// + for(Document document : cursor) + { + QRecord record = documentToRecord(table, document); + queryOutput.addRecord(record); + + if(queryInput.getAsyncJobCallback().wasCancelRequested()) + { + LOG.info("Breaking query job, as requested."); + break; + } + } + + return (queryOutput); + } + catch(Exception e) + { + /* + if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout()) + { + setQueryStatFirstResultTime(); + throw (new QUserFacingException("Query timed out.")); + } + + if(isCancelled) + { + throw (new QUserFacingException("Query was cancelled.")); + } + */ + + LOG.warn("Error executing query", e); + throw new QException("Error executing query", e); + } + finally + { + if(mongoClientContainer != null) + { + mongoClientContainer.closeIfNeeded(); + } + } + } + +} diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBTransaction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBTransaction.java new file mode 100644 index 00000000..5e367c84 --- /dev/null +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBTransaction.java @@ -0,0 +1,215 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb.actions; + + +import java.time.Duration; +import java.time.Instant; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; +import com.mongodb.client.ClientSession; +import com.mongodb.client.MongoClient; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** MongoDB implementation of backend transaction. + ** + ** Stores a mongoClient and clientSession. + ** + ** Also keeps track of if the specific mongo backend being used supports transactions, + ** as, it appears that single-node instances do not, and they throw errors if + ** you try to do transaction operations in them... This is configured by the + ** corresponding field in the backend metaData. + *******************************************************************************/ +public class MongoDBTransaction extends QBackendTransaction +{ + private static final QLogger LOG = QLogger.getLogger(MongoDBTransaction.class); + + private boolean transactionsSupported; + private MongoClient mongoClient; + private ClientSession clientSession; + + private Instant openedAt = Instant.now(); + private Integer logSlowTransactionSeconds = null; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public MongoDBTransaction(MongoDBBackendMetaData backend, MongoClient mongoClient) + { + this.transactionsSupported = backend.getTransactionsSupported(); + ClientSession clientSession = mongoClient.startSession(); + + if(transactionsSupported) + { + clientSession.startTransaction(); + } + + String propertyName = "qqq.mongodb.logSlowTransactionSeconds"; + try + { + logSlowTransactionSeconds = Integer.parseInt(System.getProperty(propertyName, "10")); + } + catch(Exception e) + { + LOG.debug("Error reading property [" + propertyName + "] value as integer", e); + } + + this.mongoClient = mongoClient; + this.clientSession = clientSession; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void commit() throws QException + { + try + { + Instant commitAt = Instant.now(); + + Duration duration = Duration.between(openedAt, commitAt); + if(logSlowTransactionSeconds != null && duration.compareTo(Duration.ofSeconds(logSlowTransactionSeconds)) > 0) + { + LOG.info("Committing long-running transaction", logPair("durationSeconds", duration.getSeconds())); + } + else + { + LOG.debug("Committing transaction"); + } + + if(transactionsSupported) + { + this.clientSession.commitTransaction(); + LOG.debug("Commit complete"); + } + else + { + LOG.debug("Request to commit, but transactions not supported in this mongodb backend"); + } + } + catch(Exception e) + { + LOG.error("Error committing transaction", e); + throw new QException("Error committing transaction: " + e.getMessage(), e); + } + finally + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // reset this - as after one commit, the transaction is essentially re-opened for any future statements that run on it // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + openedAt = Instant.now(); + if(transactionsSupported) + { + this.clientSession.startTransaction(); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void rollback() throws QException + { + try + { + if(transactionsSupported) + { + LOG.info("Rolling back transaction"); + this.clientSession.abortTransaction(); + LOG.info("Rollback complete"); + } + else + { + LOG.debug("Request to rollback, but transactions not supported in this mongodb backend"); + } + } + catch(Exception e) + { + LOG.error("Error rolling back transaction", e); + throw new QException("Error rolling back transaction: " + e.getMessage(), e); + } + finally + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // reset this - as after one commit, the transaction is essentially re-opened for any future statements that run on it // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + openedAt = Instant.now(); + if(transactionsSupported) + { + this.clientSession.startTransaction(); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void close() + { + try + { + this.clientSession.close(); + this.mongoClient.close(); + } + catch(Exception e) + { + LOG.error("Error closing connection - possible mongo connection leak", e); + } + } + + + + /******************************************************************************* + ** Getter for mongoClient + ** + *******************************************************************************/ + public MongoClient getMongoClient() + { + return mongoClient; + } + + + + /******************************************************************************* + ** Getter for clientSession + ** + *******************************************************************************/ + public ClientSession getClientSession() + { + return clientSession; + } +} diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateAction.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateAction.java new file mode 100644 index 00000000..3642b6f3 --- /dev/null +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateAction.java @@ -0,0 +1,166 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb.actions; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.actions.tables.helpers.UpdateActionRecordSplitHelper; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.ListingHash; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; +import com.mongodb.client.result.UpdateResult; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.bson.types.ObjectId; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class MongoDBUpdateAction extends AbstractMongoDBAction implements UpdateInterface +{ + private static final QLogger LOG = QLogger.getLogger(MongoDBUpdateAction.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public UpdateOutput execute(UpdateInput updateInput) throws QException + { + MongoClientContainer mongoClientContainer = null; + QTableMetaData table = updateInput.getTable(); + String backendTableName = getBackendTableName(table); + MongoDBBackendMetaData backend = (MongoDBBackendMetaData) updateInput.getBackend(); + + UpdateActionRecordSplitHelper updateActionRecordSplitHelper = new UpdateActionRecordSplitHelper(); + updateActionRecordSplitHelper.init(updateInput); + + UpdateOutput rs = new UpdateOutput(); + rs.setRecords(updateActionRecordSplitHelper.getOutputRecords()); + + if(!updateActionRecordSplitHelper.getHaveAnyWithoutErrors()) + { + LOG.info("Exiting early - all records have some error."); + return (rs); + } + + try + { + mongoClientContainer = openClient(backend, updateInput.getTransaction()); + MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName()); + MongoCollection collection = database.getCollection(backendTableName); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // process each distinct list of fields being updated (e.g., each different SQL statement) // + ///////////////////////////////////////////////////////////////////////////////////////////// + ListingHash, QRecord> recordsByFieldBeingUpdated = updateActionRecordSplitHelper.getRecordsByFieldBeingUpdated(); + for(Map.Entry, List> entry : recordsByFieldBeingUpdated.entrySet()) + { + updateRecordsWithMatchingListOfFields(updateInput, mongoClientContainer, collection, table, entry.getValue(), entry.getKey()); + } + } + catch(Exception e) + { + throw new QException("Error executing update: " + e.getMessage(), e); + } + finally + { + if(mongoClientContainer != null) + { + mongoClientContainer.closeIfNeeded(); + } + } + + return (rs); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void updateRecordsWithMatchingListOfFields(UpdateInput updateInput, MongoClientContainer mongoClientContainer, MongoCollection collection, QTableMetaData table, List recordList, List fieldsBeingUpdated) + { + boolean allAreTheSame = UpdateActionRecordSplitHelper.areAllValuesBeingUpdatedTheSame(updateInput, recordList, fieldsBeingUpdated); + if(allAreTheSame) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if all records w/ this set of fields have the same values, we can do 1 big updateMany on the whole list // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + updateRecordsWithMatchingValuesAndFields(mongoClientContainer, collection, table, recordList, fieldsBeingUpdated); + } + else + { + ///////////////////////////////////////////////////////////////////////// + // else, if not all are being updated the same, then update one-by-one // + ///////////////////////////////////////////////////////////////////////// + for(QRecord record : recordList) + { + updateRecordsWithMatchingValuesAndFields(mongoClientContainer, collection, table, List.of(record), fieldsBeingUpdated); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void updateRecordsWithMatchingValuesAndFields(MongoClientContainer mongoClientContainer, MongoCollection collection, QTableMetaData table, List recordList, List fieldsBeingUpdated) + { + QRecord firstRecord = recordList.get(0); + List ids = recordList.stream().map(r -> new ObjectId(r.getValueString("id"))).toList(); + Bson filter = Filters.in("_id", ids); + + List updates = new ArrayList<>(); + for(String fieldName : fieldsBeingUpdated) + { + QFieldMetaData field = table.getField(fieldName); + String fieldBackendName = getFieldBackendName(field); + updates.add(Updates.set(fieldBackendName, firstRecord.getValue(fieldName))); + } + Bson changes = Updates.combine(updates); + + //////////////////////////////////////////////////////// + // todo - system property to control (like print-sql) // + //////////////////////////////////////////////////////// + // LOG.debug(filter, changes); + + UpdateResult updateResult = collection.updateMany(mongoClientContainer.getMongoSession(), filter, changes); + // todo - anything with the output?? + } + +} diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/model/metadata/MongoDBBackendMetaData.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/model/metadata/MongoDBBackendMetaData.java new file mode 100644 index 00000000..2a6d769b --- /dev/null +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/model/metadata/MongoDBBackendMetaData.java @@ -0,0 +1,343 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb.model.metadata; + + +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule; + + +/******************************************************************************* + ** Meta-data to provide details of a MongoDB backend (e.g., connection params) + *******************************************************************************/ +public class MongoDBBackendMetaData extends QBackendMetaData +{ + private String host; + private Integer port; + private String databaseName; + private String username; + private String password; + private String authSourceDatabase; + private String urlSuffix; + + private boolean transactionsSupported = true; + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public MongoDBBackendMetaData() + { + super(); + setBackendType(MongoDBBackendModule.class); + } + + + + /******************************************************************************* + ** Fluent setter, override to help fluent flows + *******************************************************************************/ + @Override + public MongoDBBackendMetaData withName(String name) + { + setName(name); + return this; + } + + + + /******************************************************************************* + ** Getter for host + ** + *******************************************************************************/ + public String getHost() + { + return host; + } + + + + /******************************************************************************* + ** Setter for host + ** + *******************************************************************************/ + public void setHost(String host) + { + this.host = host; + } + + + + /******************************************************************************* + ** Fluent Setter for host + ** + *******************************************************************************/ + public MongoDBBackendMetaData withHost(String host) + { + this.host = host; + return (this); + } + + + + /******************************************************************************* + ** Getter for port + ** + *******************************************************************************/ + public Integer getPort() + { + return port; + } + + + + /******************************************************************************* + ** Setter for port + ** + *******************************************************************************/ + public void setPort(Integer port) + { + this.port = port; + } + + + + /******************************************************************************* + ** Fluent Setter for port + ** + *******************************************************************************/ + public MongoDBBackendMetaData withPort(Integer port) + { + this.port = port; + return (this); + } + + + + /******************************************************************************* + ** Getter for username + ** + *******************************************************************************/ + public String getUsername() + { + return username; + } + + + + /******************************************************************************* + ** Setter for username + ** + *******************************************************************************/ + public void setUsername(String username) + { + this.username = username; + } + + + + /******************************************************************************* + ** Fluent Setter for username + ** + *******************************************************************************/ + public MongoDBBackendMetaData withUsername(String username) + { + this.username = username; + return (this); + } + + + + /******************************************************************************* + ** Getter for password + ** + *******************************************************************************/ + public String getPassword() + { + return password; + } + + + + /******************************************************************************* + ** Setter for password + ** + *******************************************************************************/ + public void setPassword(String password) + { + this.password = password; + } + + + + /******************************************************************************* + ** Fluent Setter for password + ** + *******************************************************************************/ + public MongoDBBackendMetaData withPassword(String password) + { + this.password = password; + return (this); + } + + + + /******************************************************************************* + ** Called by the QInstanceEnricher - to do backend-type-specific enrichments. + ** Original use case is: reading secrets into fields (e.g., passwords). + *******************************************************************************/ + @Override + public void enrich() + { + super.enrich(); + QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter(); + username = interpreter.interpret(username); + password = interpreter.interpret(password); + } + + + + /******************************************************************************* + ** Getter for urlSuffix + *******************************************************************************/ + public String getUrlSuffix() + { + return (this.urlSuffix); + } + + + + /******************************************************************************* + ** Setter for urlSuffix + *******************************************************************************/ + public void setUrlSuffix(String urlSuffix) + { + this.urlSuffix = urlSuffix; + } + + + + /******************************************************************************* + ** Fluent setter for urlSuffix + *******************************************************************************/ + public MongoDBBackendMetaData withUrlSuffix(String urlSuffix) + { + this.urlSuffix = urlSuffix; + return (this); + } + + + + /******************************************************************************* + ** Getter for databaseName + *******************************************************************************/ + public String getDatabaseName() + { + return (this.databaseName); + } + + + + /******************************************************************************* + ** Setter for databaseName + *******************************************************************************/ + public void setDatabaseName(String databaseName) + { + this.databaseName = databaseName; + } + + + + /******************************************************************************* + ** Fluent setter for databaseName + *******************************************************************************/ + public MongoDBBackendMetaData withDatabaseName(String databaseName) + { + this.databaseName = databaseName; + return (this); + } + + + + /******************************************************************************* + ** Getter for transactionsSupported + *******************************************************************************/ + public boolean getTransactionsSupported() + { + return (this.transactionsSupported); + } + + + + /******************************************************************************* + ** Setter for transactionsSupported + *******************************************************************************/ + public void setTransactionsSupported(boolean transactionsSupported) + { + this.transactionsSupported = transactionsSupported; + } + + + + /******************************************************************************* + ** Fluent setter for transactionsSupported + *******************************************************************************/ + public MongoDBBackendMetaData withTransactionsSupported(boolean transactionsSupported) + { + this.transactionsSupported = transactionsSupported; + return (this); + } + + + + /******************************************************************************* + ** Getter for authSourceDatabase + *******************************************************************************/ + public String getAuthSourceDatabase() + { + return (this.authSourceDatabase); + } + + + + /******************************************************************************* + ** Setter for authSourceDatabase + *******************************************************************************/ + public void setAuthSourceDatabase(String authSourceDatabase) + { + this.authSourceDatabase = authSourceDatabase; + } + + + + /******************************************************************************* + ** Fluent setter for authSourceDatabase + *******************************************************************************/ + public MongoDBBackendMetaData withAuthSourceDatabase(String authSourceDatabase) + { + this.authSourceDatabase = authSourceDatabase; + return (this); + } + +} diff --git a/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/model/metadata/MongoDBTableBackendDetails.java b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/model/metadata/MongoDBTableBackendDetails.java new file mode 100644 index 00000000..0fa0c381 --- /dev/null +++ b/qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/model/metadata/MongoDBTableBackendDetails.java @@ -0,0 +1,81 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule; + + +/******************************************************************************* + ** Extension of QTableBackendDetails, with details specific to a MongoDB table. + *******************************************************************************/ +public class MongoDBTableBackendDetails extends QTableBackendDetails +{ + private String tableName; + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public MongoDBTableBackendDetails() + { + super(); + setBackendType(MongoDBBackendModule.class); + } + + + + /******************************************************************************* + ** Getter for tableName + ** + *******************************************************************************/ + public String getTableName() + { + return tableName; + } + + + + /******************************************************************************* + ** Setter for tableName + ** + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent Setter for tableName + ** + *******************************************************************************/ + public MongoDBTableBackendDetails withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + +} diff --git a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/BaseTest.java b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/BaseTest.java new file mode 100644 index 00000000..38488a60 --- /dev/null +++ b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/BaseTest.java @@ -0,0 +1,76 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb; + + +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + + +/******************************************************************************* + ** Base for all tests in this module + *******************************************************************************/ +public class BaseTest +{ + private static final QLogger LOG = QLogger.getLogger(BaseTest.class); + + + + /******************************************************************************* + ** init the QContext with the instance from TestUtils and a new session + *******************************************************************************/ + @BeforeEach + void baseBeforeEach() + { + QContext.init(TestUtils.defineInstance(), new QSession()); + } + + + + /******************************************************************************* + ** clear the QContext + *******************************************************************************/ + @AfterEach + void baseAfterEach() + { + QContext.clear(); + } + + + + /******************************************************************************* + ** if needed, re-initialize the QInstance in context. + *******************************************************************************/ + protected static void reInitInstanceInContext(QInstance qInstance) + { + if(qInstance.equals(QContext.getQInstance())) + { + LOG.warn("Unexpected condition - the same qInstance that is already in the QContext was passed into reInit. You probably want a new QInstance object instance."); + } + QContext.init(qInstance, new QSession()); + } + +} diff --git a/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/TestUtils.java b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/TestUtils.java new file mode 100644 index 00000000..7f188131 --- /dev/null +++ b/qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/TestUtils.java @@ -0,0 +1,145 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.mongodb; + + +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; +import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBTableBackendDetails; + + +/******************************************************************************* + ** Test Utils class for this module + *******************************************************************************/ +public class TestUtils +{ + public static final String DEFAULT_BACKEND_NAME = "default"; + + public static final String TABLE_NAME_PERSON = "personTable"; + + public static final String SECURITY_KEY_STORE_ALL_ACCESS = "storeAllAccess"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static void primeTestDatabase(String sqlFileName) throws Exception + { + /* + ConnectionManager connectionManager = new ConnectionManager(); + try(Connection connection = connectionManager.getConnection(TestUtils.defineBackend())) + { + InputStream primeTestDatabaseSqlStream = RDBMSActionTest.class.getResourceAsStream("/" + sqlFileName); + assertNotNull(primeTestDatabaseSqlStream); + List lines = (List) IOUtils.readLines(primeTestDatabaseSqlStream); + lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList(); + String joinedSQL = String.join("\n", lines); + for(String sql : joinedSQL.split(";")) + { + QueryManager.executeUpdate(connection, sql); + } + } + */ + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QInstance defineInstance() + { + QInstance qInstance = new QInstance(); + qInstance.addBackend(defineBackend()); + qInstance.addTable(defineTablePerson()); + qInstance.setAuthentication(defineAuthentication()); + return (qInstance); + } + + + + /******************************************************************************* + ** Define the authentication used in standard tests - using 'mock' type. + ** + *******************************************************************************/ + public static QAuthenticationMetaData defineAuthentication() + { + return new QAuthenticationMetaData() + .withName("mock") + .withType(QAuthenticationType.MOCK); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static MongoDBBackendMetaData defineBackend() + { + return (new MongoDBBackendMetaData() + .withName(DEFAULT_BACKEND_NAME) + .withHost("localhost") + .withPort(27017) + .withUsername("ctliveuser") + .withPassword("uoaKOIjfk23h8lozK983L") + .withAuthSourceDatabase("admin") + .withDatabaseName("testDatabase") + /*.withUrlSuffix("?tls=true&tlsCAFile=global-bundle.pem&retryWrites=false")*/); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineTablePerson() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSON) + .withLabel("Person") + .withRecordLabelFormat("%s %s") + .withRecordLabelFields("firstName", "lastName") + .withBackendName(DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.STRING).withBackendName("_id")) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)) + .withField(new QFieldMetaData("firstName", QFieldType.STRING)) + .withField(new QFieldMetaData("lastName", QFieldType.STRING)) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE)) + .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN)) + .withField(new QFieldMetaData("annualSalary", QFieldType.DECIMAL)) + .withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER)) + .withField(new QFieldMetaData("homeTown", QFieldType.STRING)) + .withBackendDetails(new MongoDBTableBackendDetails() + .withTableName("testTable")); + } + +}