QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-17 20:50:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-781 Initial build of mongodb backend module

Browse Source
This commit is contained in:
darin.kelkhoff
2024-01-08 20:00:57 -06:00
parent 68911190fa
commit f5c4c12388
16 changed files with 2942 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pom.xml
View File

@ -33,6 +33,7 @@
<module>qqq-backend-module-api</module>
<module>qqq-backend-module-filesystem</module>
<module>qqq-backend-module-rdbms</module>
<module>qqq-backend-module-mongodb</module>
<module>qqq-language-support-javascript</module>
<module>qqq-middleware-picocli</module>
<module>qqq-middleware-javalin</module>

120
qqq-backend-module-mongodb/pom.xml Normal file
View File

@ -0,0 +1,120 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ QQQ - Low-code Application Framework for Engineers.
~ Copyright (C) 2021-2024. Kingsrook, LLC
~ 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
~ contact@kingsrook.com
~ https://github.com/Kingsrook/
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU Affero General Public License as
~ published by the Free Software Foundation, either version 3 of the
~ License, or (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU Affero General Public License for more details.
~
~ You should have received a copy of the GNU Affero General Public License
~ along with this program. If not, see <https://www.gnu.org/licenses/>.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>qqq-backend-module-mongodb</artifactId>
<parent>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-parent-project</artifactId>
<version>${revision}</version>
</parent>
<properties>
<!-- props specifically to this module -->
<!-- none at this time -->
</properties>
<dependencies>
<!-- other qqq modules deps -->
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-core</artifactId>
<version>${revision}</version>
</dependency>
<!-- 3rd party deps specifically for this module -->
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver-sync</artifactId>
<version>4.11.1</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>2.17.1</version>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mongodb</artifactId>
<version>1.19.3</version>
<scope>test</scope>
</dependency>
<!-- Common deps for all qqq modules -->
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>${plugin.shade.phase}</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

168
qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/MongoDBBackendModule.java Normal file
View File

@ -0,0 +1,168 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
import com.kingsrook.qqq.backend.module.mongodb.actions.AbstractMongoDBAction;
import com.kingsrook.qqq.backend.module.mongodb.actions.MongoClientContainer;
import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBAggregateAction;
import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBCountAction;
import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBDeleteAction;
import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBInsertAction;
import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBQueryAction;
import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBTransaction;
import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBUpdateAction;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBTableBackendDetails;
/*******************************************************************************
** QQQ Backend module for working with MongoDB
*******************************************************************************/
public class MongoDBBackendModule implements QBackendModuleInterface
{
static
{
QBackendModuleDispatcher.registerBackendModule(new MongoDBBackendModule());
}
/*******************************************************************************
** Method where a backend module must be able to provide its type (name).
*******************************************************************************/
public String getBackendType()
{
return ("mongodb");
}
/*******************************************************************************
** Method to identify the class used for backend meta data for this module.
*******************************************************************************/
@Override
public Class<? extends QBackendMetaData> getBackendMetaDataClass()
{
return (MongoDBBackendMetaData.class);
}
/*******************************************************************************
** Method to identify the class used for table-backend details for this module.
*******************************************************************************/
@Override
public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
{
return (MongoDBTableBackendDetails.class);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public CountInterface getCountInterface()
{
return (new MongoDBCountAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QueryInterface getQueryInterface()
{
return (new MongoDBQueryAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public InsertInterface getInsertInterface()
{
return (new MongoDBInsertAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public UpdateInterface getUpdateInterface()
{
return (new MongoDBUpdateAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public DeleteInterface getDeleteInterface()
{
return (new MongoDBDeleteAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public AggregateInterface getAggregateInterface()
{
return (new MongoDBAggregateAction());
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public QBackendTransaction openTransaction(AbstractTableActionInput input)
{
MongoDBBackendMetaData backend = (MongoDBBackendMetaData) input.getBackend();
MongoClientContainer mongoClientContainer = new AbstractMongoDBAction().openClient(backend, null);
return (new MongoDBTransaction(backend, mongoClientContainer.getMongoClient()));
}
}

541
qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java Normal file
View File

@ -0,0 +1,541 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb.actions;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.regex.Pattern;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBTableBackendDetails;
import com.mongodb.ConnectionString;
import com.mongodb.MongoClientSettings;
import com.mongodb.MongoCredential;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.model.Filters;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.bson.types.ObjectId;
/*******************************************************************************
** Base class for all mongoDB module actions.
*******************************************************************************/
public class AbstractMongoDBAction
{
private static final QLogger LOG = QLogger.getLogger(AbstractMongoDBAction.class);
/*******************************************************************************
** Open a MongoDB Client / session -- re-using the one in the input transaction
** if it is present.
*******************************************************************************/
public MongoClientContainer openClient(MongoDBBackendMetaData backend, QBackendTransaction transaction)
{
if(transaction instanceof MongoDBTransaction mongoDBTransaction)
{
//////////////////////////////////////////////////////////////////////////////////////////
// re-use the connection from the transaction (indicating false in last parameter here) //
//////////////////////////////////////////////////////////////////////////////////////////
return (new MongoClientContainer(mongoDBTransaction.getMongoClient(), mongoDBTransaction.getClientSession(), false));
}
ConnectionString connectionString = new ConnectionString("mongodb://" + backend.getHost() + ":" + backend.getPort() + "/");
MongoCredential credential = MongoCredential.createCredential(backend.getUsername(), backend.getAuthSourceDatabase(), backend.getPassword().toCharArray());
MongoClientSettings settings = MongoClientSettings.builder()
////////////////////////////////////////////////
// is this needed, what, for a cluster maybe? //
////////////////////////////////////////////////
// .applyToClusterSettings(builder -> builder.hosts(seeds))
.applyConnectionString(connectionString)
.credential(credential)
.build();
MongoClient mongoClient = MongoClients.create(settings);
////////////////////////////////////////////////////////////////////////////
// indicate that this connection was newly opened via the true param here //
////////////////////////////////////////////////////////////////////////////
return (new MongoClientContainer(mongoClient, mongoClient.startSession(), true));
}
/*******************************************************************************
** Get the name to use for a field in the mongoDB, from the fieldMetaData.
**
** That is, field.backendName if set -- else, field.name
*******************************************************************************/
protected String getFieldBackendName(QFieldMetaData field)
{
if(field.getBackendName() != null)
{
return (field.getBackendName());
}
return (field.getName());
}
/*******************************************************************************
** Get the name to use for a table in the mongoDB, from the table's backendDetails.
**
** else, the table's name.
*******************************************************************************/
protected String getBackendTableName(QTableMetaData table)
{
if(table.getBackendDetails() != null)
{
String backendTableName = ((MongoDBTableBackendDetails) table.getBackendDetails()).getTableName();
if(StringUtils.hasContent(backendTableName))
{
return (backendTableName);
}
}
return table.getName();
}
/*******************************************************************************
**
*******************************************************************************/
protected int getPageSize()
{
return (1000);
}
/*******************************************************************************
** Convert a mongodb document to a QRecord.
*******************************************************************************/
protected QRecord documentToRecord(QTableMetaData table, Document document)
{
QRecord record = new QRecord();
record.setTableName(table.getName());
///////////////////////////////////////////////////////////////////////////
// todo - this - or iterate over the values in the document?? //
// seems like, maybe, this is an attribute in the table-backend-details? //
///////////////////////////////////////////////////////////////////////////
Map<String, Serializable> values = record.getValues();
for(QFieldMetaData field : table.getFields().values())
{
String fieldBackendName = getFieldBackendName(field);
Object value = document.get(fieldBackendName);
String fieldName = field.getName();
setValue(values, fieldName, value);
}
return (record);
}
/*******************************************************************************
** Recursive helper method to put a value in a map - where mongodb documents
** are recursively expanded, and types are mapped to QQQ expectations.
*******************************************************************************/
private void setValue(Map<String, Serializable> values, String fieldName, Object value)
{
if(value instanceof ObjectId objectId)
{
values.put(fieldName, objectId.toString());
}
else if(value instanceof java.util.Date date)
{
values.put(fieldName, date.toInstant());
}
else if(value instanceof Document document)
{
LinkedHashMap<String, Serializable> subValues = new LinkedHashMap<>();
values.put(fieldName, subValues);
for(String subFieldName : document.keySet())
{
Object subValue = document.get(subFieldName);
setValue(subValues, subFieldName, subValue);
}
}
else if(value instanceof Serializable s)
{
values.put(fieldName, s);
}
else if(value != null)
{
values.put(fieldName, String.valueOf(value));
}
else
{
values.put(fieldName, null);
}
}
/*******************************************************************************
** Convert a QRecord to a mongodb document.
*******************************************************************************/
protected Document recordToDocument(QTableMetaData table, QRecord record)
{
Document document = new Document();
///////////////////////////////////////////////////////////////////////////
// todo - this - or iterate over the values in the record?? //
// seems like, maybe, this is an attribute in the table-backend-details? //
///////////////////////////////////////////////////////////////////////////
for(QFieldMetaData field : table.getFields().values())
{
if(field.getName().equals(table.getPrimaryKeyField()) && record.getValue(field.getName()) == null)
{
////////////////////////////////////
// let mongodb client generate id //
////////////////////////////////////
continue;
}
String fieldBackendName = getFieldBackendName(field);
document.append(fieldBackendName, record.getValue(field.getName()));
}
return (document);
}
/*******************************************************************************
** Convert QQueryFilter to Bson search query document - including security
** for the table if needed.
*******************************************************************************/
protected Bson makeSearchQueryDocument(QTableMetaData table, QQueryFilter filter) throws QException
{
Bson searchQueryWithoutSecurity = makeSearchQueryDocumentWithoutSecurity(table, filter);
QQueryFilter securityFilter = makeSecurityQueryFilter(table);
if(!securityFilter.hasAnyCriteria())
{
return (searchQueryWithoutSecurity);
}
Bson searchQueryForSecurity = makeSearchQueryDocumentWithoutSecurity(table, securityFilter);
return (Filters.and(searchQueryWithoutSecurity, searchQueryForSecurity));
}
/*******************************************************************************
** Build a QQueryFilter to apply record-level security to the query.
** Note, it may be empty, if there are no lock fields, or all are all-access.
**
** Originally copied from RDBMS module... should this be shared?
** and/or, how big of a re-write did that get in the joins-enhancements branch...
*******************************************************************************/
private QQueryFilter makeSecurityQueryFilter(QTableMetaData table) throws QException
{
QQueryFilter securityFilter = new QQueryFilter();
securityFilter.setBooleanOperator(QQueryFilter.BooleanOperator.AND);
for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
{
addSubFilterForRecordSecurityLock(QContext.getQInstance(), QContext.getQSession(), table, securityFilter, recordSecurityLock, null, table.getName(), false);
}
return (securityFilter);
}
/*******************************************************************************
** Helper for makeSecuritySearchQuery.
**
** Originally copied from RDBMS module... should this be shared?
** and/or, how big of a re-write did that get in the joins-enhancements branch...
*******************************************************************************/
private static void addSubFilterForRecordSecurityLock(QInstance instance, QSession session, QTableMetaData table, QQueryFilter securityFilter, RecordSecurityLock recordSecurityLock, JoinsContext joinsContext, String tableNameOrAlias, boolean isOuter) throws QException
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// check if the key type has an all-access key, and if so, if it's set to true for the current user/session //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()))
{
if(session.hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
{
///////////////////////////////////////////////////////////////////////////////
// if we have all-access on this key, then we don't need a criterion for it. //
///////////////////////////////////////////////////////////////////////////////
return;
}
}
///////////////////////////////////////////////////////////////////////////////////////
// some differences from RDBMS here, due to not yet having joins support in mongo... //
///////////////////////////////////////////////////////////////////////////////////////
// String fieldName = tableNameOrAlias + "." + recordSecurityLock.getFieldName();
String fieldName = recordSecurityLock.getFieldName();
if(CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()))
{
throw (new QException("Security locks in mongodb with joinNameChain is not yet supported"));
// fieldName = recordSecurityLock.getFieldName();
}
///////////////////////////////////////////////////////////////////////////////////////////
// else - get the key values from the session and decide what kind of criterion to build //
///////////////////////////////////////////////////////////////////////////////////////////
QQueryFilter lockFilter = new QQueryFilter();
List<QFilterCriteria> lockCriteria = new ArrayList<>();
lockFilter.setCriteria(lockCriteria);
QFieldType type = QFieldType.INTEGER;
try
{
if(joinsContext == null)
{
type = table.getField(fieldName).getType();
}
else
{
JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(fieldName);
type = fieldAndTableNameOrAlias.field().getType();
}
}
catch(Exception e)
{
LOG.debug("Error getting field type... Trying Integer", e);
}
List<Serializable> securityKeyValues = session.getSecurityKeyValues(recordSecurityLock.getSecurityKeyType(), type);
if(CollectionUtils.nullSafeIsEmpty(securityKeyValues))
{
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// handle user with no values -- they can only see null values, and only iff the lock's null-value behavior is ALLOW //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(RecordSecurityLock.NullValueBehavior.ALLOW.equals(recordSecurityLock.getNullValueBehavior()))
{
lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
}
else
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// else, if no user/session values, and null-value behavior is deny, then setup a FALSE condition, to allow no rows. //
// todo - make some explicit contradiction here - maybe even avoid running the whole query - as you're not allowed ANY records //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IN, Collections.emptyList()));
}
}
else
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// else, if user/session has some values, build an IN rule - //
// noting that if the lock's null-value behavior is ALLOW, then we actually want IS_NULL_OR_IN, not just IN //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(RecordSecurityLock.NullValueBehavior.ALLOW.equals(recordSecurityLock.getNullValueBehavior()))
{
lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IS_NULL_OR_IN, securityKeyValues));
}
else
{
lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IN, securityKeyValues));
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if this field is on the outer side of an outer join, then if we do a straight filter on it, then we're basically //
// nullifying the outer join... so for an outer join use-case, OR the security field criteria with a primary-key IS NULL //
// which will make missing rows from the join be found. //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(isOuter)
{
lockFilter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
lockFilter.addCriteria(new QFilterCriteria(tableNameOrAlias + "." + table.getPrimaryKeyField(), QCriteriaOperator.IS_BLANK));
}
securityFilter.addSubFilter(lockFilter);
}
/*******************************************************************************
** w/o considering security, just map a QQueryFilter to a Bson searchQuery.
*******************************************************************************/
@SuppressWarnings("checkstyle:Indentation")
private Bson makeSearchQueryDocumentWithoutSecurity(QTableMetaData table, QQueryFilter filter)
{
if(filter == null || !filter.hasAnyCriteria())
{
return (new Document());
}
List<Bson> criteriaFilters = new ArrayList<>();
for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria()))
{
List<Serializable> values = criteria.getValues() == null ? new ArrayList<>() : new ArrayList<>(criteria.getValues());
QFieldMetaData field = table.getField(criteria.getFieldName());
String fieldBackendName = getFieldBackendName(field);
if(field.getName().equals(table.getPrimaryKeyField()))
{
ListIterator<Serializable> iterator = values.listIterator();
while(iterator.hasNext())
{
Serializable value = iterator.next();
iterator.set(new ObjectId(String.valueOf(value)));
}
}
Serializable value0 = values.get(0);
criteriaFilters.add(switch(criteria.getOperator())
{
case EQUALS -> Filters.eq(fieldBackendName, value0);
case NOT_EQUALS -> Filters.ne(fieldBackendName, value0);
case NOT_EQUALS_OR_IS_NULL -> Filters.or(
Filters.eq(fieldBackendName, null),
Filters.ne(fieldBackendName, value0)
);
case IN -> filterIn(fieldBackendName, values);
case NOT_IN -> Filters.not(filterIn(fieldBackendName, values));
case IS_NULL_OR_IN -> Filters.or(
Filters.eq(fieldBackendName, null),
filterIn(fieldBackendName, values)
);
case LIKE -> filterRegex(fieldBackendName, null, ValueUtils.getValueAsString(value0).replaceAll("%", ".*"), null);
case NOT_LIKE -> Filters.not(filterRegex(fieldBackendName, null, ValueUtils.getValueAsString(value0).replaceAll("%", ".*"), null));
case STARTS_WITH -> filterRegex(fieldBackendName, null, value0, ".*");
case ENDS_WITH -> filterRegex(fieldBackendName, ".*", value0, null);
case CONTAINS -> filterRegex(fieldBackendName, ".*", value0, ".*");
case NOT_STARTS_WITH -> Filters.not(filterRegex(fieldBackendName, null, value0, ".*"));
case NOT_ENDS_WITH -> Filters.not(filterRegex(fieldBackendName, ".*", value0, null));
case NOT_CONTAINS -> Filters.not(filterRegex(fieldBackendName, ".*", value0, ".*"));
case LESS_THAN -> Filters.lt(fieldBackendName, value0);
case LESS_THAN_OR_EQUALS -> Filters.lte(fieldBackendName, value0);
case GREATER_THAN -> Filters.gt(fieldBackendName, value0);
case GREATER_THAN_OR_EQUALS -> Filters.gte(fieldBackendName, value0);
case IS_BLANK -> filterIsBlank(fieldBackendName);
case IS_NOT_BLANK -> Filters.not(filterIsBlank(fieldBackendName));
case BETWEEN -> filterBetween(fieldBackendName, values);
case NOT_BETWEEN -> Filters.not(filterBetween(fieldBackendName, values));
});
}
/////////////////////////////////////
// recursively process sub-filters //
/////////////////////////////////////
if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
{
for(QQueryFilter subFilter : filter.getSubFilters())
{
criteriaFilters.add(makeSearchQueryDocumentWithoutSecurity(table, subFilter));
}
}
Bson bson = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? Filters.and(criteriaFilters) : Filters.or(criteriaFilters);
return bson;
}
/*******************************************************************************
** build a bson filter doing a regex (e.g., for LIKE, STARTS_WITH, etc)
*******************************************************************************/
private Bson filterRegex(String fieldBackendName, String prefix, Serializable mainRegex, String suffix)
{
if(prefix == null)
{
prefix = "";
}
if(suffix == null)
{
suffix = "";
}
String fullRegex = prefix + Pattern.quote(ValueUtils.getValueAsString(mainRegex) + suffix);
return (Filters.regex(fieldBackendName, Pattern.compile(fullRegex)));
}
/*******************************************************************************
** build a bson filter doing IN
*******************************************************************************/
private static Bson filterIn(String fieldBackendName, List<Serializable> values)
{
return Filters.in(fieldBackendName, values);
}
/*******************************************************************************
** build a bson filter doing BETWEEN
*******************************************************************************/
private static Bson filterBetween(String fieldBackendName, List<Serializable> values)
{
return Filters.and(
Filters.gte(fieldBackendName, values.get(0)),
Filters.lte(fieldBackendName, values.get(1))
);
}
/*******************************************************************************
** build a bson filter doing BLANK (null or == "")
*******************************************************************************/
private static Bson filterIsBlank(String fieldBackendName)
{
return Filters.or(
Filters.eq(fieldBackendName, null),
Filters.eq(fieldBackendName, "")
);
}
}

158
qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoClientContainer.java Normal file
View File

@ -0,0 +1,158 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb.actions;
import com.mongodb.client.ClientSession;
import com.mongodb.client.MongoClient;
/*******************************************************************************
** Wrapper around a MongoClient, ClientSession, and a boolean to help signal
** where it was opened (e.g., so you know if you need to close it yourself, or
** if it came from someone else (e.g., via an input transaction)).
*******************************************************************************/
public class MongoClientContainer
{
private MongoClient mongoClient;
private ClientSession mongoSession;
private boolean needToClose;
/*******************************************************************************
**
*******************************************************************************/
public MongoClientContainer(MongoClient mongoClient, ClientSession mongoSession, boolean needToClose)
{
this.mongoClient = mongoClient;
this.mongoSession = mongoSession;
this.needToClose = needToClose;
}
/*******************************************************************************
** Getter for mongoClient
*******************************************************************************/
public MongoClient getMongoClient()
{
return (this.mongoClient);
}
/*******************************************************************************
** Setter for mongoClient
*******************************************************************************/
public void setMongoClient(MongoClient mongoClient)
{
this.mongoClient = mongoClient;
}
/*******************************************************************************
** Fluent setter for mongoClient
*******************************************************************************/
public MongoClientContainer withMongoClient(MongoClient mongoClient)
{
this.mongoClient = mongoClient;
return (this);
}
/*******************************************************************************
** Getter for mongoSession
*******************************************************************************/
public ClientSession getMongoSession()
{
return (this.mongoSession);
}
/*******************************************************************************
** Setter for mongoSession
*******************************************************************************/
public void setMongoSession(ClientSession mongoSession)
{
this.mongoSession = mongoSession;
}
/*******************************************************************************
** Fluent setter for mongoSession
*******************************************************************************/
public MongoClientContainer withMongoSession(ClientSession mongoSession)
{
this.mongoSession = mongoSession;
return (this);
}
/*******************************************************************************
** Getter for needToClose
*******************************************************************************/
public boolean getNeedToClose()
{
return (this.needToClose);
}
/*******************************************************************************
** Setter for needToClose
*******************************************************************************/
public void setNeedToClose(boolean needToClose)
{
this.needToClose = needToClose;
}
/*******************************************************************************
** Fluent setter for needToClose
*******************************************************************************/
public MongoClientContainer withNeedToClose(boolean needToClose)
{
this.needToClose = needToClose;
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public void closeIfNeeded()
{
if(needToClose)
{
mongoSession.close();
mongoClient.close();
}
}
}

251
qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateAction.java Normal file
View File

@ -0,0 +1,251 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb.actions;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
import com.mongodb.client.AggregateIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Accumulators;
import com.mongodb.client.model.Aggregates;
import com.mongodb.client.model.BsonField;
import org.bson.Document;
import org.bson.conversions.Bson;
/*******************************************************************************
**
*******************************************************************************/
public class MongoDBAggregateAction extends AbstractMongoDBAction implements AggregateInterface
{
private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class);
// todo? private ActionTimeoutHelper actionTimeoutHelper;
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("checkstyle:indentation")
public AggregateOutput execute(AggregateInput aggregateInput) throws QException
{
MongoClientContainer mongoClientContainer = null;
try
{
AggregateOutput aggregateOutput = new AggregateOutput();
QTableMetaData table = aggregateInput.getTable();
String backendTableName = getBackendTableName(table);
MongoDBBackendMetaData backend = (MongoDBBackendMetaData) aggregateInput.getBackend();
mongoClientContainer = openClient(backend, null); // todo - aggregate input has no transaction!?
MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName());
MongoCollection<Document> collection = database.getCollection(backendTableName);
QQueryFilter filter = aggregateInput.getFilter();
Bson searchQuery = makeSearchQueryDocument(table, filter);
/////////////////////////////////////////////////////////////////////////
// we have to submit a list of BSON objects to the aggregate function. //
// the first one is the search query //
// second is the group-by stuff, which we'll explain as we build it //
/////////////////////////////////////////////////////////////////////////
List<Bson> bsonList = new ArrayList<>();
bsonList.add(Aggregates.match(searchQuery));
//////////////////////////////////////////////////////////////////////////////////////
// if there are group-by fields, then we need to build a document with those fields //
// not sure what the whole name, $name is, but, go go mongo //
//////////////////////////////////////////////////////////////////////////////////////
Document groupValueDocument = new Document();
if(CollectionUtils.nullSafeHasContents(aggregateInput.getGroupBys()))
{
for(GroupBy groupBy : aggregateInput.getGroupBys())
{
String name = getFieldBackendName(table.getField(groupBy.getFieldName()));
groupValueDocument.append(name, "$" + name);
}
}
////////////////////////////////////////////////////////////////////
// next build a list of accumulator fields - for aggregate values //
////////////////////////////////////////////////////////////////////
List<BsonField> bsonFields = new ArrayList<>();
for(Aggregate aggregate : aggregateInput.getAggregates())
{
String fieldName = aggregate.getFieldName() + "_" + aggregate.getOperator().toString().toLowerCase();
String expression = "$" + getFieldBackendName(table.getField(aggregate.getFieldName()));
bsonFields.add(switch(aggregate.getOperator())
{
case COUNT -> Accumulators.sum(fieldName, 1); // count... do a sum of 1's
case COUNT_DISTINCT -> throw new QException("Count Distinct is not supported for MongoDB tables at this time.");
case SUM -> Accumulators.sum(fieldName, expression);
case MIN -> Accumulators.min(fieldName, expression);
case MAX -> Accumulators.max(fieldName, expression);
case AVG -> Accumulators.avg(fieldName, expression);
});
}
///////////////////////////////////////////////////////////////////////////////////
// add the group-by fields and the aggregates in the group stage of the pipeline //
///////////////////////////////////////////////////////////////////////////////////
bsonList.add(Aggregates.group(groupValueDocument, bsonFields));
//////////////////////////////////////////////
// if there are any order-bys, add them too //
//////////////////////////////////////////////
if(filter != null && CollectionUtils.nullSafeHasContents(filter.getOrderBys()))
{
Document sortValue = new Document();
for(QFilterOrderBy orderBy : filter.getOrderBys())
{
String fieldName;
if(orderBy instanceof QFilterOrderByAggregate orderByAggregate)
{
Aggregate aggregate = orderByAggregate.getAggregate();
fieldName = aggregate.getFieldName() + "_" + aggregate.getOperator().toString().toLowerCase();
}
else if(orderBy instanceof QFilterOrderByGroupBy orderByGroupBy)
{
fieldName = "_id." + getFieldBackendName(table.getField(orderByGroupBy.getGroupBy().getFieldName()));
}
else
{
///////////////////////////////////////////////////
// does this happen? should it be "_id." if so? //
///////////////////////////////////////////////////
fieldName = getFieldBackendName(table.getField(orderBy.getFieldName()));
}
sortValue.append(fieldName, orderBy.getIsAscending() ? 1 : -1);
}
bsonList.add(new Document("$sort", sortValue));
}
////////////////////////////////////////////////////////
// todo - system property to control (like print-sql) //
////////////////////////////////////////////////////////
// LOG.debug(bsonList.toString());
///////////////////////////
// execute the aggregate //
///////////////////////////
AggregateIterable<Document> aggregates = collection.aggregate(mongoClientContainer.getMongoSession(), bsonList);
List<AggregateResult> results = new ArrayList<>();
aggregateOutput.setResults(results);
/////////////////////
// process results //
/////////////////////
for(Document document : aggregates)
{
AggregateResult result = new AggregateResult();
results.add(result);
////////////////////////////////////////////////////////////////
// get group by values (if there are any) out of the document //
////////////////////////////////////////////////////////////////
for(GroupBy groupBy : CollectionUtils.nonNullList(aggregateInput.getGroupBys()))
{
Document idDocument = (Document) document.get("_id");
Object value = idDocument.get(groupBy.getFieldName());
result.withGroupByValue(groupBy, ValueUtils.getValueAsFieldType(groupBy.getType(), value));
}
//////////////////////////////////////////
// get aggregate values out of document //
//////////////////////////////////////////
for(Aggregate aggregate : aggregateInput.getAggregates())
{
QFieldMetaData field = table.getField(aggregate.getFieldName());
QFieldType fieldType = aggregate.getFieldType();
if(fieldType == null)
{
fieldType = field.getType();
}
if(fieldType.equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG)))
{
fieldType = QFieldType.DECIMAL;
}
Object value = document.get(aggregate.getFieldName() + "_" + aggregate.getOperator().toString().toLowerCase());
result.withAggregateValue(aggregate, ValueUtils.getValueAsFieldType(fieldType, value));
}
}
return (aggregateOutput);
}
catch(Exception e)
{
/*
if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout())
{
setCountStatFirstResultTime();
throw (new QUserFacingException("Aggregate timed out."));
}
if(isCancelled)
{
throw (new QUserFacingException("Aggregate was cancelled."));
}
*/
LOG.warn("Error executing aggregate", e);
throw new QException("Error executing aggregate", e);
}
finally
{
if(mongoClientContainer != null)
{
mongoClientContainer.closeIfNeeded();
}
}
}
}

119
qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountAction.java Normal file
View File

@ -0,0 +1,119 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb.actions;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
import com.mongodb.client.AggregateIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Accumulators;
import com.mongodb.client.model.Aggregates;
import org.bson.Document;
import org.bson.conversions.Bson;
/*******************************************************************************
**
*******************************************************************************/
public class MongoDBCountAction extends AbstractMongoDBAction implements CountInterface
{
private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class);
// todo? private ActionTimeoutHelper actionTimeoutHelper;
/*******************************************************************************
**
*******************************************************************************/
public CountOutput execute(CountInput countInput) throws QException
{
MongoClientContainer mongoClientContainer = null;
try
{
CountOutput countOutput = new CountOutput();
QTableMetaData table = countInput.getTable();
String backendTableName = getBackendTableName(table);
MongoDBBackendMetaData backend = (MongoDBBackendMetaData) countInput.getBackend();
mongoClientContainer = openClient(backend, null); // todo - count input has no transaction!?
MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName());
MongoCollection<Document> collection = database.getCollection(backendTableName);
QQueryFilter filter = countInput.getFilter();
Bson searchQuery = makeSearchQueryDocument(table, filter);
List<Bson> bsonList = List.of(
Aggregates.match(searchQuery),
Aggregates.group("_id", Accumulators.sum("count", 1)));
////////////////////////////////////////////////////////
// todo - system property to control (like print-sql) //
////////////////////////////////////////////////////////
// LOG.debug(bsonList.toString());
AggregateIterable<Document> aggregate = collection.aggregate(mongoClientContainer.getMongoSession(), bsonList);
Document document = aggregate.first();
countOutput.setCount(document == null ? 0 : document.get("count", Integer.class));
return (countOutput);
}
catch(Exception e)
{
/*
if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout())
{
setCountStatFirstResultTime();
throw (new QUserFacingException("Count timed out."));
}
if(isCancelled)
{
throw (new QUserFacingException("Count was cancelled."));
}
*/
LOG.warn("Error executing count", e);
throw new QException("Error executing count", e);
}
finally
{
if(mongoClientContainer != null)
{
mongoClientContainer.closeIfNeeded();
}
}
}
}

129
qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteAction.java Normal file
View File

@ -0,0 +1,129 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Filters;
import com.mongodb.client.result.DeleteResult;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.bson.types.ObjectId;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
**
*******************************************************************************/
public class MongoDBDeleteAction extends AbstractMongoDBAction implements DeleteInterface
{
private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class);
/*******************************************************************************
**
*******************************************************************************/
@Override
public boolean supportsQueryFilterInput()
{
return (true);
}
/*******************************************************************************
**
*******************************************************************************/
public DeleteOutput execute(DeleteInput deleteInput) throws QException
{
MongoClientContainer mongoClientContainer = null;
try
{
DeleteOutput deleteOutput = new DeleteOutput();
QTableMetaData table = deleteInput.getTable();
String backendTableName = getBackendTableName(table);
MongoDBBackendMetaData backend = (MongoDBBackendMetaData) deleteInput.getBackend();
mongoClientContainer = openClient(backend, deleteInput.getTransaction());
MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName());
MongoCollection<Document> collection = database.getCollection(backendTableName);
QQueryFilter queryFilter = deleteInput.getQueryFilter();
Bson searchQuery;
if(CollectionUtils.nullSafeHasContents(deleteInput.getPrimaryKeys()))
{
searchQuery = Filters.in("_id", deleteInput.getPrimaryKeys().stream().map(id -> new ObjectId(ValueUtils.getValueAsString(id))).toList());
}
else if(queryFilter != null && queryFilter.hasAnyCriteria())
{
QQueryFilter filter = queryFilter;
searchQuery = makeSearchQueryDocument(table, filter);
}
else
{
LOG.info("Missing both primary keys and a search filter in delete request - exiting with noop", logPair("tableName", table.getName()));
return (deleteOutput);
}
////////////////////////////////////////////////////////
// todo - system property to control (like print-sql) //
////////////////////////////////////////////////////////
// LOG.debug(searchQuery);
DeleteResult deleteResult = collection.deleteMany(mongoClientContainer.getMongoSession(), searchQuery);
deleteOutput.setDeletedRecordCount((int) deleteResult.getDeletedCount());
//////////////////////////////////////////////////////////////////////////
// todo any way to get records with errors or warnings for deleteOutput //
//////////////////////////////////////////////////////////////////////////
return (deleteOutput);
}
catch(Exception e)
{
LOG.warn("Error executing delete", e);
throw new QException("Error executing delete", e);
}
finally
{
if(mongoClientContainer != null)
{
mongoClientContainer.closeIfNeeded();
}
}
}
}

266
qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertAction.java Normal file
View File

@ -0,0 +1,266 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb.actions;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.result.InsertManyResult;
import org.bson.BsonValue;
import org.bson.Document;
/*******************************************************************************
**
*******************************************************************************/
public class MongoDBInsertAction extends AbstractMongoDBAction implements InsertInterface
{
private static final QLogger LOG = QLogger.getLogger(MongoDBInsertAction.class);
/*******************************************************************************
**
*******************************************************************************/
public InsertOutput execute(InsertInput insertInput) throws QException
{
MongoClientContainer mongoClientContainer = null;
InsertOutput rs = new InsertOutput();
List<QRecord> outputRecords = new ArrayList<>();
rs.setRecords(outputRecords);
try
{
QTableMetaData table = insertInput.getTable();
String backendTableName = getBackendTableName(table);
MongoDBBackendMetaData backend = (MongoDBBackendMetaData) insertInput.getBackend();
mongoClientContainer = openClient(backend, insertInput.getTransaction());
MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName());
MongoCollection<Document> collection = database.getCollection(backendTableName);
//////////////////////////
// todo - transaction?! //
//////////////////////////
///////////////////////////////////////////////////////////////////////////
// page over input record list (assuming some size of batch is too big?) //
///////////////////////////////////////////////////////////////////////////
for(List<QRecord> page : CollectionUtils.getPages(insertInput.getRecords(), getPageSize()))
{
//////////////////////////////////////////////////////////////////
// build list of documents from records w/o errors in this page //
//////////////////////////////////////////////////////////////////
List<Document> documentList = new ArrayList<>();
for(QRecord record : page)
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
continue;
}
documentList.add(recordToDocument(table, record));
}
/////////////////////////////////////
// skip pages that were all errors //
/////////////////////////////////////
if(documentList.isEmpty())
{
continue;
}
////////////////////////////////////////////////////////
// todo - system property to control (like print-sql) //
////////////////////////////////////////////////////////
// LOG.debug(documentList);
///////////////////////////////////////////////
// actually do the insert //
// todo - how are errors returned by mongo?? //
///////////////////////////////////////////////
InsertManyResult insertManyResult = collection.insertMany(mongoClientContainer.getMongoSession(), documentList);
/////////////////////////////////
// put ids on inserted records //
/////////////////////////////////
int index = 0;
for(QRecord record : page)
{
QRecord outputRecord = new QRecord(record);
rs.addRecord(outputRecord);
if(CollectionUtils.nullSafeIsEmpty(record.getErrors()))
{
BsonValue insertedId = insertManyResult.getInsertedIds().get(index++);
String idString = insertedId.asObjectId().getValue().toString();
outputRecord.setValue(table.getPrimaryKeyField(), idString);
}
}
}
}
catch(Exception e)
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
finally
{
if(mongoClientContainer != null)
{
mongoClientContainer.closeIfNeeded();
}
}
return (rs);
/*
try
{
List<QFieldMetaData> insertableFields = table.getFields().values().stream()
.filter(field -> !field.getName().equals("id")) // todo - intent here is to avoid non-insertable fields.
.toList();
String columns = insertableFields.stream()
.map(f -> "`" + getColumnName(f) + "`")
.collect(Collectors.joining(", "));
String questionMarks = insertableFields.stream()
.map(x -> "?")
.collect(Collectors.joining(", "));
List<QRecord> outputRecords = new ArrayList<>();
rs.setRecords(outputRecords);
Connection connection;
boolean needToCloseConnection = false;
if(insertInput.getTransaction() != null && insertInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
{
connection = rdbmsTransaction.getConnection();
}
else
{
connection = getConnection(insertInput);
needToCloseConnection = true;
}
try
{
for(List<QRecord> page : CollectionUtils.getPages(insertInput.getRecords(), QueryManager.PAGE_SIZE))
{
String tableName = escapeIdentifier(getTableName(table));
StringBuilder sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(columns).append(") VALUES");
List<Object> params = new ArrayList<>();
int recordIndex = 0;
//////////////////////////////////////////////////////
// for each record in the page: //
// - if it has errors, skip it //
// - else add a "(?,?,...,?)," clause to the INSERT //
// - then add all fields into the params list //
//////////////////////////////////////////////////////
for(QRecord record : page)
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
continue;
}
if(recordIndex++ > 0)
{
sql.append(",");
}
sql.append("(").append(questionMarks).append(")");
for(QFieldMetaData field : insertableFields)
{
Serializable value = record.getValue(field.getName());
value = scrubValue(field, value);
params.add(value);
}
}
////////////////////////////////////////////////////////////////////////////////////////
// if all records had errors, copy them to the output, and continue w/o running query //
////////////////////////////////////////////////////////////////////////////////////////
if(recordIndex == 0)
{
for(QRecord record : page)
{
QRecord outputRecord = new QRecord(record);
outputRecords.add(outputRecord);
}
continue;
}
Long mark = System.currentTimeMillis();
///////////////////////////////////////////////////////////
// execute the insert, then foreach record in the input, //
// add it to the output, and set its generated id too. //
///////////////////////////////////////////////////////////
// todo sql customization - can edit sql and/or param list
// todo - non-serial-id style tables
// todo - other generated values, e.g., createDate... maybe need to re-select?
List<Integer> idList = QueryManager.executeInsertForGeneratedIds(connection, sql.toString(), params);
int index = 0;
for(QRecord record : page)
{
QRecord outputRecord = new QRecord(record);
outputRecords.add(outputRecord);
if(CollectionUtils.nullSafeIsEmpty(record.getErrors()))
{
Integer id = idList.get(index++);
outputRecord.setValue(table.getPrimaryKeyField(), id);
}
}
logSQL(sql, params, mark);
}
}
finally
{
if(needToCloseConnection)
{
connection.close();
}
}
return rs;
}
catch(Exception e)
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
*/
}
}

163
qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryAction.java Normal file
View File

@ -0,0 +1,163 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb.actions;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import org.bson.Document;
import org.bson.conversions.Bson;
/*******************************************************************************
**
*******************************************************************************/
public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryInterface
{
private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class);
// todo? private ActionTimeoutHelper actionTimeoutHelper;
/*******************************************************************************
**
*******************************************************************************/
public QueryOutput execute(QueryInput queryInput) throws QException
{
MongoClientContainer mongoClientContainer = null;
try
{
QueryOutput queryOutput = new QueryOutput(queryInput);
QTableMetaData table = queryInput.getTable();
String backendTableName = getBackendTableName(table);
MongoDBBackendMetaData backend = (MongoDBBackendMetaData) queryInput.getBackend();
mongoClientContainer = openClient(backend, queryInput.getTransaction());
MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName());
MongoCollection<Document> collection = database.getCollection(backendTableName);
/////////////////////////
// set up filter/query //
/////////////////////////
QQueryFilter filter = queryInput.getFilter();
Bson searchQuery = makeSearchQueryDocument(table, filter);
////////////////////////////////////////////////////////
// todo - system property to control (like print-sql) //
////////////////////////////////////////////////////////
// LOG.debug(searchQuery);
////////////////////////////////////////////////////////////
// create cursor - further adjustments to it still follow //
////////////////////////////////////////////////////////////
FindIterable<Document> cursor = collection.find(mongoClientContainer.getMongoSession(), searchQuery);
///////////////////////////////////
// add a sort operator if needed //
///////////////////////////////////
if(filter != null && CollectionUtils.nullSafeHasContents(filter.getOrderBys()))
{
Document sortDocument = new Document();
for(QFilterOrderBy orderBy : filter.getOrderBys())
{
String fieldBackendName = getFieldBackendName(table.getField(orderBy.getFieldName()));
sortDocument.put(fieldBackendName, orderBy.getIsAscending() ? 1 : -1);
}
cursor.sort(sortDocument);
}
////////////////////////
// apply skip & limit //
////////////////////////
if(filter != null)
{
if(filter.getSkip() != null)
{
cursor.skip(filter.getSkip());
}
if(filter.getLimit() != null)
{
cursor.limit(filter.getLimit());
}
}
////////////////////////////////////////////
// iterate over results, building records //
////////////////////////////////////////////
for(Document document : cursor)
{
QRecord record = documentToRecord(table, document);
queryOutput.addRecord(record);
if(queryInput.getAsyncJobCallback().wasCancelRequested())
{
LOG.info("Breaking query job, as requested.");
break;
}
}
return (queryOutput);
}
catch(Exception e)
{
/*
if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout())
{
setQueryStatFirstResultTime();
throw (new QUserFacingException("Query timed out."));
}
if(isCancelled)
{
throw (new QUserFacingException("Query was cancelled."));
}
*/
LOG.warn("Error executing query", e);
throw new QException("Error executing query", e);
}
finally
{
if(mongoClientContainer != null)
{
mongoClientContainer.closeIfNeeded();
}
}
}
}

215
qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBTransaction.java Normal file
View File

@ -0,0 +1,215 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb.actions;
import java.time.Duration;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
import com.mongodb.client.ClientSession;
import com.mongodb.client.MongoClient;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** MongoDB implementation of backend transaction.
**
** Stores a mongoClient and clientSession.
**
** Also keeps track of if the specific mongo backend being used supports transactions,
** as, it appears that single-node instances do not, and they throw errors if
** you try to do transaction operations in them... This is configured by the
** corresponding field in the backend metaData.
*******************************************************************************/
public class MongoDBTransaction extends QBackendTransaction
{
private static final QLogger LOG = QLogger.getLogger(MongoDBTransaction.class);
private boolean transactionsSupported;
private MongoClient mongoClient;
private ClientSession clientSession;
private Instant openedAt = Instant.now();
private Integer logSlowTransactionSeconds = null;
/*******************************************************************************
**
*******************************************************************************/
public MongoDBTransaction(MongoDBBackendMetaData backend, MongoClient mongoClient)
{
this.transactionsSupported = backend.getTransactionsSupported();
ClientSession clientSession = mongoClient.startSession();
if(transactionsSupported)
{
clientSession.startTransaction();
}
String propertyName = "qqq.mongodb.logSlowTransactionSeconds";
try
{
logSlowTransactionSeconds = Integer.parseInt(System.getProperty(propertyName, "10"));
}
catch(Exception e)
{
LOG.debug("Error reading property [" + propertyName + "] value as integer", e);
}
this.mongoClient = mongoClient;
this.clientSession = clientSession;
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void commit() throws QException
{
try
{
Instant commitAt = Instant.now();
Duration duration = Duration.between(openedAt, commitAt);
if(logSlowTransactionSeconds != null && duration.compareTo(Duration.ofSeconds(logSlowTransactionSeconds)) > 0)
{
LOG.info("Committing long-running transaction", logPair("durationSeconds", duration.getSeconds()));
}
else
{
LOG.debug("Committing transaction");
}
if(transactionsSupported)
{
this.clientSession.commitTransaction();
LOG.debug("Commit complete");
}
else
{
LOG.debug("Request to commit, but transactions not supported in this mongodb backend");
}
}
catch(Exception e)
{
LOG.error("Error committing transaction", e);
throw new QException("Error committing transaction: " + e.getMessage(), e);
}
finally
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// reset this - as after one commit, the transaction is essentially re-opened for any future statements that run on it //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
openedAt = Instant.now();
if(transactionsSupported)
{
this.clientSession.startTransaction();
}
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void rollback() throws QException
{
try
{
if(transactionsSupported)
{
LOG.info("Rolling back transaction");
this.clientSession.abortTransaction();
LOG.info("Rollback complete");
}
else
{
LOG.debug("Request to rollback, but transactions not supported in this mongodb backend");
}
}
catch(Exception e)
{
LOG.error("Error rolling back transaction", e);
throw new QException("Error rolling back transaction: " + e.getMessage(), e);
}
finally
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// reset this - as after one commit, the transaction is essentially re-opened for any future statements that run on it //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
openedAt = Instant.now();
if(transactionsSupported)
{
this.clientSession.startTransaction();
}
}
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void close()
{
try
{
this.clientSession.close();
this.mongoClient.close();
}
catch(Exception e)
{
LOG.error("Error closing connection - possible mongo connection leak", e);
}
}
/*******************************************************************************
** Getter for mongoClient
**
*******************************************************************************/
public MongoClient getMongoClient()
{
return mongoClient;
}
/*******************************************************************************
** Getter for clientSession
**
*******************************************************************************/
public ClientSession getClientSession()
{
return clientSession;
}
}

166
qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateAction.java Normal file
View File

@ -0,0 +1,166 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb.actions;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.UpdateActionRecordSplitHelper;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.ListingHash;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.Updates;
import com.mongodb.client.result.UpdateResult;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.bson.types.ObjectId;
/*******************************************************************************
**
*******************************************************************************/
public class MongoDBUpdateAction extends AbstractMongoDBAction implements UpdateInterface
{
private static final QLogger LOG = QLogger.getLogger(MongoDBUpdateAction.class);
/*******************************************************************************
**
*******************************************************************************/
public UpdateOutput execute(UpdateInput updateInput) throws QException
{
MongoClientContainer mongoClientContainer = null;
QTableMetaData table = updateInput.getTable();
String backendTableName = getBackendTableName(table);
MongoDBBackendMetaData backend = (MongoDBBackendMetaData) updateInput.getBackend();
UpdateActionRecordSplitHelper updateActionRecordSplitHelper = new UpdateActionRecordSplitHelper();
updateActionRecordSplitHelper.init(updateInput);
UpdateOutput rs = new UpdateOutput();
rs.setRecords(updateActionRecordSplitHelper.getOutputRecords());
if(!updateActionRecordSplitHelper.getHaveAnyWithoutErrors())
{
LOG.info("Exiting early - all records have some error.");
return (rs);
}
try
{
mongoClientContainer = openClient(backend, updateInput.getTransaction());
MongoDatabase database = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName());
MongoCollection<Document> collection = database.getCollection(backendTableName);
/////////////////////////////////////////////////////////////////////////////////////////////
// process each distinct list of fields being updated (e.g., each different SQL statement) //
/////////////////////////////////////////////////////////////////////////////////////////////
ListingHash<List<String>, QRecord> recordsByFieldBeingUpdated = updateActionRecordSplitHelper.getRecordsByFieldBeingUpdated();
for(Map.Entry<List<String>, List<QRecord>> entry : recordsByFieldBeingUpdated.entrySet())
{
updateRecordsWithMatchingListOfFields(updateInput, mongoClientContainer, collection, table, entry.getValue(), entry.getKey());
}
}
catch(Exception e)
{
throw new QException("Error executing update: " + e.getMessage(), e);
}
finally
{
if(mongoClientContainer != null)
{
mongoClientContainer.closeIfNeeded();
}
}
return (rs);
}
/*******************************************************************************
**
*******************************************************************************/
private void updateRecordsWithMatchingListOfFields(UpdateInput updateInput, MongoClientContainer mongoClientContainer, MongoCollection<Document> collection, QTableMetaData table, List<QRecord> recordList, List<String> fieldsBeingUpdated)
{
boolean allAreTheSame = UpdateActionRecordSplitHelper.areAllValuesBeingUpdatedTheSame(updateInput, recordList, fieldsBeingUpdated);
if(allAreTheSame)
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if all records w/ this set of fields have the same values, we can do 1 big updateMany on the whole list //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
updateRecordsWithMatchingValuesAndFields(mongoClientContainer, collection, table, recordList, fieldsBeingUpdated);
}
else
{
/////////////////////////////////////////////////////////////////////////
// else, if not all are being updated the same, then update one-by-one //
/////////////////////////////////////////////////////////////////////////
for(QRecord record : recordList)
{
updateRecordsWithMatchingValuesAndFields(mongoClientContainer, collection, table, List.of(record), fieldsBeingUpdated);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private void updateRecordsWithMatchingValuesAndFields(MongoClientContainer mongoClientContainer, MongoCollection<Document> collection, QTableMetaData table, List<QRecord> recordList, List<String> fieldsBeingUpdated)
{
QRecord firstRecord = recordList.get(0);
List<ObjectId> ids = recordList.stream().map(r -> new ObjectId(r.getValueString("id"))).toList();
Bson filter = Filters.in("_id", ids);
List<Bson> updates = new ArrayList<>();
for(String fieldName : fieldsBeingUpdated)
{
QFieldMetaData field = table.getField(fieldName);
String fieldBackendName = getFieldBackendName(field);
updates.add(Updates.set(fieldBackendName, firstRecord.getValue(fieldName)));
}
Bson changes = Updates.combine(updates);
////////////////////////////////////////////////////////
// todo - system property to control (like print-sql) //
////////////////////////////////////////////////////////
// LOG.debug(filter, changes);
UpdateResult updateResult = collection.updateMany(mongoClientContainer.getMongoSession(), filter, changes);
// todo - anything with the output??
}
}

343
qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/model/metadata/MongoDBBackendMetaData.java Normal file
View File

@ -0,0 +1,343 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb.model.metadata;
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule;
/*******************************************************************************
** Meta-data to provide details of a MongoDB backend (e.g., connection params)
*******************************************************************************/
public class MongoDBBackendMetaData extends QBackendMetaData
{
private String host;
private Integer port;
private String databaseName;
private String username;
private String password;
private String authSourceDatabase;
private String urlSuffix;
private boolean transactionsSupported = true;
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public MongoDBBackendMetaData()
{
super();
setBackendType(MongoDBBackendModule.class);
}
/*******************************************************************************
** Fluent setter, override to help fluent flows
*******************************************************************************/
@Override
public MongoDBBackendMetaData withName(String name)
{
setName(name);
return this;
}
/*******************************************************************************
** Getter for host
**
*******************************************************************************/
public String getHost()
{
return host;
}
/*******************************************************************************
** Setter for host
**
*******************************************************************************/
public void setHost(String host)
{
this.host = host;
}
/*******************************************************************************
** Fluent Setter for host
**
*******************************************************************************/
public MongoDBBackendMetaData withHost(String host)
{
this.host = host;
return (this);
}
/*******************************************************************************
** Getter for port
**
*******************************************************************************/
public Integer getPort()
{
return port;
}
/*******************************************************************************
** Setter for port
**
*******************************************************************************/
public void setPort(Integer port)
{
this.port = port;
}
/*******************************************************************************
** Fluent Setter for port
**
*******************************************************************************/
public MongoDBBackendMetaData withPort(Integer port)
{
this.port = port;
return (this);
}
/*******************************************************************************
** Getter for username
**
*******************************************************************************/
public String getUsername()
{
return username;
}
/*******************************************************************************
** Setter for username
**
*******************************************************************************/
public void setUsername(String username)
{
this.username = username;
}
/*******************************************************************************
** Fluent Setter for username
**
*******************************************************************************/
public MongoDBBackendMetaData withUsername(String username)
{
this.username = username;
return (this);
}
/*******************************************************************************
** Getter for password
**
*******************************************************************************/
public String getPassword()
{
return password;
}
/*******************************************************************************
** Setter for password
**
*******************************************************************************/
public void setPassword(String password)
{
this.password = password;
}
/*******************************************************************************
** Fluent Setter for password
**
*******************************************************************************/
public MongoDBBackendMetaData withPassword(String password)
{
this.password = password;
return (this);
}
/*******************************************************************************
** Called by the QInstanceEnricher - to do backend-type-specific enrichments.
** Original use case is: reading secrets into fields (e.g., passwords).
*******************************************************************************/
@Override
public void enrich()
{
super.enrich();
QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
username = interpreter.interpret(username);
password = interpreter.interpret(password);
}
/*******************************************************************************
** Getter for urlSuffix
*******************************************************************************/
public String getUrlSuffix()
{
return (this.urlSuffix);
}
/*******************************************************************************
** Setter for urlSuffix
*******************************************************************************/
public void setUrlSuffix(String urlSuffix)
{
this.urlSuffix = urlSuffix;
}
/*******************************************************************************
** Fluent setter for urlSuffix
*******************************************************************************/
public MongoDBBackendMetaData withUrlSuffix(String urlSuffix)
{
this.urlSuffix = urlSuffix;
return (this);
}
/*******************************************************************************
** Getter for databaseName
*******************************************************************************/
public String getDatabaseName()
{
return (this.databaseName);
}
/*******************************************************************************
** Setter for databaseName
*******************************************************************************/
public void setDatabaseName(String databaseName)
{
this.databaseName = databaseName;
}
/*******************************************************************************
** Fluent setter for databaseName
*******************************************************************************/
public MongoDBBackendMetaData withDatabaseName(String databaseName)
{
this.databaseName = databaseName;
return (this);
}
/*******************************************************************************
** Getter for transactionsSupported
*******************************************************************************/
public boolean getTransactionsSupported()
{
return (this.transactionsSupported);
}
/*******************************************************************************
** Setter for transactionsSupported
*******************************************************************************/
public void setTransactionsSupported(boolean transactionsSupported)
{
this.transactionsSupported = transactionsSupported;
}
/*******************************************************************************
** Fluent setter for transactionsSupported
*******************************************************************************/
public MongoDBBackendMetaData withTransactionsSupported(boolean transactionsSupported)
{
this.transactionsSupported = transactionsSupported;
return (this);
}
/*******************************************************************************
** Getter for authSourceDatabase
*******************************************************************************/
public String getAuthSourceDatabase()
{
return (this.authSourceDatabase);
}
/*******************************************************************************
** Setter for authSourceDatabase
*******************************************************************************/
public void setAuthSourceDatabase(String authSourceDatabase)
{
this.authSourceDatabase = authSourceDatabase;
}
/*******************************************************************************
** Fluent setter for authSourceDatabase
*******************************************************************************/
public MongoDBBackendMetaData withAuthSourceDatabase(String authSourceDatabase)
{
this.authSourceDatabase = authSourceDatabase;
return (this);
}
}

81
qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/model/metadata/MongoDBTableBackendDetails.java Normal file
View File

@ -0,0 +1,81 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb.model.metadata;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule;
/*******************************************************************************
** Extension of QTableBackendDetails, with details specific to a MongoDB table.
*******************************************************************************/
public class MongoDBTableBackendDetails extends QTableBackendDetails
{
private String tableName;
/*******************************************************************************
** Default Constructor.
*******************************************************************************/
public MongoDBTableBackendDetails()
{
super();
setBackendType(MongoDBBackendModule.class);
}
/*******************************************************************************
** Getter for tableName
**
*******************************************************************************/
public String getTableName()
{
return tableName;
}
/*******************************************************************************
** Setter for tableName
**
*******************************************************************************/
public void setTableName(String tableName)
{
this.tableName = tableName;
}
/*******************************************************************************
** Fluent Setter for tableName
**
*******************************************************************************/
public MongoDBTableBackendDetails withTableName(String tableName)
{
this.tableName = tableName;
return (this);
}
}

76
qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/BaseTest.java Normal file
View File

@ -0,0 +1,76 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
/*******************************************************************************
** Base for all tests in this module
*******************************************************************************/
public class BaseTest
{
private static final QLogger LOG = QLogger.getLogger(BaseTest.class);
/*******************************************************************************
** init the QContext with the instance from TestUtils and a new session
*******************************************************************************/
@BeforeEach
void baseBeforeEach()
{
QContext.init(TestUtils.defineInstance(), new QSession());
}
/*******************************************************************************
** clear the QContext
*******************************************************************************/
@AfterEach
void baseAfterEach()
{
QContext.clear();
}
/*******************************************************************************
** if needed, re-initialize the QInstance in context.
*******************************************************************************/
protected static void reInitInstanceInContext(QInstance qInstance)
{
if(qInstance.equals(QContext.getQInstance()))
{
LOG.warn("Unexpected condition - the same qInstance that is already in the QContext was passed into reInit. You probably want a new QInstance object instance.");
}
QContext.init(qInstance, new QSession());
}
}

145
qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/TestUtils.java Normal file
View File

@ -0,0 +1,145 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.mongodb;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBTableBackendDetails;
/*******************************************************************************
** Test Utils class for this module
*******************************************************************************/
public class TestUtils
{
public static final String DEFAULT_BACKEND_NAME = "default";
public static final String TABLE_NAME_PERSON = "personTable";
public static final String SECURITY_KEY_STORE_ALL_ACCESS = "storeAllAccess";
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public static void primeTestDatabase(String sqlFileName) throws Exception
{
/*
ConnectionManager connectionManager = new ConnectionManager();
try(Connection connection = connectionManager.getConnection(TestUtils.defineBackend()))
{
InputStream primeTestDatabaseSqlStream = RDBMSActionTest.class.getResourceAsStream("/" + sqlFileName);
assertNotNull(primeTestDatabaseSqlStream);
List<String> lines = (List<String>) IOUtils.readLines(primeTestDatabaseSqlStream);
lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList();
String joinedSQL = String.join("\n", lines);
for(String sql : joinedSQL.split(";"))
{
QueryManager.executeUpdate(connection, sql);
}
}
*/
}
/*******************************************************************************
**
*******************************************************************************/
public static QInstance defineInstance()
{
QInstance qInstance = new QInstance();
qInstance.addBackend(defineBackend());
qInstance.addTable(defineTablePerson());
qInstance.setAuthentication(defineAuthentication());
return (qInstance);
}
/*******************************************************************************
** Define the authentication used in standard tests - using 'mock' type.
**
*******************************************************************************/
public static QAuthenticationMetaData defineAuthentication()
{
return new QAuthenticationMetaData()
.withName("mock")
.withType(QAuthenticationType.MOCK);
}
/*******************************************************************************
**
*******************************************************************************/
public static MongoDBBackendMetaData defineBackend()
{
return (new MongoDBBackendMetaData()
.withName(DEFAULT_BACKEND_NAME)
.withHost("localhost")
.withPort(27017)
.withUsername("ctliveuser")
.withPassword("uoaKOIjfk23h8lozK983L")
.withAuthSourceDatabase("admin")
.withDatabaseName("testDatabase")
/*.withUrlSuffix("?tls=true&tlsCAFile=global-bundle.pem&retryWrites=false")*/);
}
/*******************************************************************************
**
*******************************************************************************/
public static QTableMetaData defineTablePerson()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON)
.withLabel("Person")
.withRecordLabelFormat("%s %s")
.withRecordLabelFields("firstName", "lastName")
.withBackendName(DEFAULT_BACKEND_NAME)
.withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.STRING).withBackendName("_id"))
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))
.withField(new QFieldMetaData("firstName", QFieldType.STRING))
.withField(new QFieldMetaData("lastName", QFieldType.STRING))
.withField(new QFieldMetaData("birthDate", QFieldType.DATE))
.withField(new QFieldMetaData("email", QFieldType.STRING))
.withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN))
.withField(new QFieldMetaData("annualSalary", QFieldType.DECIMAL))
.withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER))
.withField(new QFieldMetaData("homeTown", QFieldType.STRING))
.withBackendDetails(new MongoDBTableBackendDetails()
.withTableName("testTable"));
}
}